This is an automated email from the ASF dual-hosted git repository. changchen pushed a commit to branch feature/ansi-ci-baseline in repository https://gitbox.apache.org/repos/asf/gluten.git
commit 9113b152e9b93e7ae7f876de180d379d54932333 Author: Chang chen <[email protected]> AuthorDate: Mon Mar 30 10:26:11 2026 +0000 ci(ansi): simplify CastWithAnsiOffSuite + add Spark 4.0 ANSI CI jobs - Remove all redundant overrides from GlutenCastWithAnsiOffSuite (cast(), testGluten rewrites, UDT registration) for both spark40 and spark41, keeping only beforeAll + GlutenTestsTrait mixin - Remove corresponding excludes from VeloxTestSettings for both versions - Enable GlutenCastWithAnsiOnSuite for spark40 - Add --jvm-arg support to run-scala-test.sh - Add Spark 4.0 backends-velox and spark-ut ANSI CI jobs to workflow Co-Authored-By: Claude Opus 4.6 <[email protected]> --- .github/workflows/velox_backend_ansi.yml | 155 ++++++++++++ dev/run-scala-test.sh | 13 + .../gluten/utils/velox/VeloxTestSettings.scala | 12 +- .../expressions/GlutenCastWithAnsiOffSuite.scala | 267 -------------------- .../gluten/utils/velox/VeloxTestSettings.scala | 10 - .../expressions/GlutenCastWithAnsiOffSuite.scala | 274 +-------------------- 6 files changed, 170 insertions(+), 561 deletions(-) diff --git a/.github/workflows/velox_backend_ansi.yml b/.github/workflows/velox_backend_ansi.yml index 51af6edb31..f5e3679064 100644 --- a/.github/workflows/velox_backend_ansi.yml +++ b/.github/workflows/velox_backend_ansi.yml @@ -242,3 +242,158 @@ jobs: **/target/*.log **/gluten-ut/**/hs_err_*.log **/gluten-ut/**/core.* + + spark-test-backends-velox-ansi-spark40: + needs: build-native-lib + runs-on: ubuntu-22.04 + env: + SPARK_TESTING: true + container: apache/gluten:centos-9-jdk17 + steps: + - uses: actions/checkout@v4 + - name: Download Native Lib + uses: actions/download-artifact@v4 + with: + name: velox-native-lib-ansi-${{github.sha}} + path: ./cpp/build/ + - name: Download Arrow Jars + uses: actions/download-artifact@v4 + with: + name: arrow-jars-ansi-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ + - name: Prepare + run: | + dnf install -y python3.11 python3.11-pip python3.11-devel && \ + alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ + alternatives --set python3 /usr/bin/python3.11 && \ + pip3 install setuptools==77.0.3 && \ + pip3 install pyspark==3.5.5 cython && \ + pip3 install pandas==2.2.3 pyarrow==20.0.0 + - name: Prepare Spark Resources + run: | + rm -rf /opt/shims/spark40 + bash .github/workflows/util/install-resources.sh 4.0 + mv /opt/shims/spark40/spark_home/assembly/target/scala-2.12 /opt/shims/spark40/spark_home/assembly/target/scala-2.13 + - name: "Spark 4.0 backends-velox Tests (ANSI ON)" + run: | + set -o pipefail + cd $GITHUB_WORKSPACE/ + export SPARK_SCALA_VERSION=2.13 + yum install -y java-17-openjdk-devel + export JAVA_HOME=/usr/lib/jvm/java-17-openjdk + export PATH=$JAVA_HOME/bin:$PATH + echo "SPARK_ANSI_SQL_MODE=$SPARK_ANSI_SQL_MODE" + java -version + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox \ + -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/ -Dspark.gluten.sql.ansiFallback.enabled=false" \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ + 2>&1 | tee backends-velox-spark40-test-output.log + - name: "Parse test results" + if: always() + run: | + echo "=========================================" + echo " backends-velox (Spark 4.0, ANSI=$SPARK_ANSI_SQL_MODE)" + echo "=========================================" + echo "" + echo "--- Test Summary ---" + grep -E "Tests run:.*Failures:|BUILD " backends-velox-spark40-test-output.log || echo "(no summary found)" + echo "" + echo "--- Failed Tests ---" + grep -B1 "<<< FAIL!" backends-velox-spark40-test-output.log || echo "(no failures)" + echo "" + echo "--- Error Tests ---" + grep -B1 "<<< ERROR!" backends-velox-spark40-test-output.log || echo "(no errors)" + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: spark-test-backends-velox-ansi-spark40-report + path: | + **/surefire-reports/TEST-*.xml + backends-velox-spark40-test-output.log + - name: Upload log files + if: always() + uses: actions/upload-artifact@v4 + with: + name: spark-test-backends-velox-ansi-spark40-logs + path: | + **/target/*.log + + spark-test-spark-ut-ansi-spark40: + needs: build-native-lib + runs-on: ubuntu-22.04 + env: + SPARK_TESTING: true + container: apache/gluten:centos-9-jdk17 + steps: + - uses: actions/checkout@v4 + - name: Download Native Lib + uses: actions/download-artifact@v4 + with: + name: velox-native-lib-ansi-${{github.sha}} + path: ./cpp/build/ + - name: Download Arrow Jars + uses: actions/download-artifact@v4 + with: + name: arrow-jars-ansi-${{github.sha}} + path: /root/.m2/repository/org/apache/arrow/ + - name: Prepare + run: | + dnf install -y python3.11 python3.11-pip python3.11-devel && \ + alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 && \ + alternatives --set python3 /usr/bin/python3.11 && \ + pip3 install setuptools==77.0.3 && \ + pip3 install pyspark==3.5.5 cython && \ + pip3 install pandas==2.2.3 pyarrow==20.0.0 + - name: Prepare Spark Resources + run: | + rm -rf /opt/shims/spark40 + bash .github/workflows/util/install-resources.sh 4.0 + mv /opt/shims/spark40/spark_home/assembly/target/scala-2.12 /opt/shims/spark40/spark_home/assembly/target/scala-2.13 + - name: "Spark 4.0 spark-ut Tests (ANSI ON)" + run: | + set -o pipefail + cd $GITHUB_WORKSPACE/ + export SPARK_SCALA_VERSION=2.13 + yum install -y java-17-openjdk-devel + export JAVA_HOME=/usr/lib/jvm/java-17-openjdk + export PATH=$JAVA_HOME/bin:$PATH + echo "SPARK_ANSI_SQL_MODE=$SPARK_ANSI_SQL_MODE" + java -version + $MVN_CMD clean test -Pspark-4.0 -Pscala-2.13 -Pjava-17 -Pbackends-velox -Pspark-ut \ + -DwildcardSuites='org.apache.spark.' -Dtest=none -DfailIfNoTests=false \ + -DargLine="-Dspark.test.home=/opt/shims/spark40/spark_home/ -Dspark.gluten.sql.ansiFallback.enabled=false" \ + -DtagsToExclude=org.apache.spark.tags.ExtendedSQLTest,org.apache.gluten.tags.UDFTest,org.apache.gluten.tags.EnhancedFeaturesTest,org.apache.gluten.tags.SkipTest \ + 2>&1 | tee spark-ut-spark40-test-output.log + - name: "Parse test results" + if: always() + run: | + echo "=========================================" + echo " spark-ut (Spark 4.0, ANSI=$SPARK_ANSI_SQL_MODE)" + echo "=========================================" + echo "" + echo "--- Test Summary ---" + grep -E "Tests run:.*Failures:|BUILD " spark-ut-spark40-test-output.log || echo "(no summary found)" + echo "" + echo "--- Failed Tests ---" + grep -B1 "<<< FAIL!" spark-ut-spark40-test-output.log || echo "(no failures)" + echo "" + echo "--- Error Tests ---" + grep -B1 "<<< ERROR!" spark-ut-spark40-test-output.log || echo "(no errors)" + - name: Upload test report + if: always() + uses: actions/upload-artifact@v4 + with: + name: spark-test-spark-ut-ansi-spark40-report + path: | + **/surefire-reports/TEST-*.xml + spark-ut-spark40-test-output.log + - name: Upload log files + if: always() + uses: actions/upload-artifact@v4 + with: + name: spark-test-spark-ut-ansi-spark40-logs + path: | + **/target/*.log + **/gluten-ut/**/hs_err_*.log + **/gluten-ut/**/core.* diff --git a/dev/run-scala-test.sh b/dev/run-scala-test.sh index 2142c17e08..06aee7b25b 100755 --- a/dev/run-scala-test.sh +++ b/dev/run-scala-test.sh @@ -215,6 +215,7 @@ Optional: --force Force Maven rebuild, bypass build cache --profile Enable Maven profiler (reports in .profiler/) --export-only Export classpath and exit (no test execution) + --jvm-arg <arg> Pass extra JVM argument to test process (repeatable) --help Show this help message Examples: @@ -234,6 +235,12 @@ Examples: -pl gluten-ut/spark40 \\ -s org.apache.spark.sql.GlutenTPCDSV1_4_PlanStabilitySuite \\ -s org.apache.spark.sql.GlutenTPCHPlanStabilitySuite + + # Run with ANSI mode (disable fallback via --jvm-arg) + $0 --jvm-arg "-Dspark.gluten.sql.ansiFallback.enabled=false" \\ + -Pjava-17,spark-4.1,scala-2.13,backends-velox,hadoop-3.3,spark-ut \\ + -pl gluten-ut/spark41 \\ + -s org.apache.spark.sql.catalyst.expressions.GlutenCastWithAnsiOffSuite EOF } @@ -348,6 +355,7 @@ EXPORT_ONLY=false ENABLE_CLEAN=false FORCE_BUILD=false USE_MVND=false +EXTRA_JVM_ARGS=() while [[ $# -gt 0 ]]; do case $1 in @@ -387,6 +395,10 @@ while [[ $# -gt 0 ]]; do EXPORT_ONLY=true shift ;; + --jvm-arg) + EXTRA_JVM_ARGS+=("$2") + shift 2 + ;; --help) print_usage exit 0 @@ -684,6 +696,7 @@ SPARK_TEST_HOME_ARG="" JAVA_ARGS=( ${JVM_ARGS} + "${EXTRA_JVM_ARGS[@]}" "-Dlog4j.configurationFile=file:${GLUTEN_HOME}/${MODULE}/src/test/resources/log4j2.properties" ${SPARK_TEST_HOME_ARG} -cp "${PATHING_JAR}" diff --git a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index 498483cb65..8def0d92e8 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -102,16 +102,6 @@ class VeloxTestSettings extends BackendTestSettings { .exclude("SPARK-45786: Decimal multiply, divide, remainder, quot") enableSuite[GlutenBitwiseExpressionsSuite] enableSuite[GlutenCastWithAnsiOffSuite] - .exclude( - "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly. - ) - // Set timezone through config. - .exclude("data type casting") - // Revised by setting timezone through config and commented unsupported cases. - .exclude("cast string to timestamp") - .exclude("cast from timestamp II") - .exclude("SPARK-36286: invalid string cast to timestamp") - .exclude("SPARK-39749: cast Decimal to string") enableSuite[GlutenTryCastSuite] .exclude( "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly. @@ -217,7 +207,7 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenBitmapExpressionUtilsSuite] enableSuite[GlutenCallMethodViaReflectionSuite] enableSuite[GlutenCanonicalizeSuite] - // TODO: 4.x enableSuite[GlutenCastWithAnsiOnSuite] // 4 failures + enableSuite[GlutenCastWithAnsiOnSuite] enableSuite[GlutenCodeGenerationSuite] enableSuite[GlutenCodeGeneratorWithInterpretedFallbackSuite] enableSuite[GlutenCollationExpressionSuite] diff --git a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala index 74c1b25ca2..291b986504 100644 --- a/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala +++ b/gluten-ut/spark40/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala @@ -17,14 +17,7 @@ package org.apache.spark.sql.catalyst.expressions import org.apache.spark.sql.GlutenTestsTrait -import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, ALL_TIMEZONES, UTC, UTC_OPT} -import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp, millisToMicros, TimeZoneUTC} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ -import org.apache.spark.util.DebuggableThreadUtils - -import java.sql.{Date, Timestamp} -import java.util.{Calendar, TimeZone} class GlutenCastWithAnsiOffSuite extends CastWithAnsiOffSuite with GlutenTestsTrait { override def beforeAll(): Unit = { @@ -35,264 +28,4 @@ class GlutenCastWithAnsiOffSuite extends CastWithAnsiOffSuite with GlutenTestsTr // In Vanilla spark, the checkEvaluation method doesn't invoke RowEncoder. conf.setConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO, true) } - - override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): Cast = { - v match { - case lit: Expression => - logDebug(s"Cast from: ${lit.dataType.typeName}, to: ${targetType.typeName}") - Cast(lit, targetType, timeZoneId) - case _ => - val lit = Literal(v) - logDebug(s"Cast from: ${lit.dataType.typeName}, to: ${targetType.typeName}") - Cast(lit, targetType, timeZoneId) - } - } - - // Register UDT For test("SPARK-32828") - UDTRegistration.register(classOf[IExampleBaseType].getName, classOf[ExampleBaseTypeUDT].getName) - UDTRegistration.register(classOf[IExampleSubType].getName, classOf[ExampleSubTypeUDT].getName) - - testGluten("missing cases - from boolean") { - (DataTypeTestUtils.numericTypeWithoutDecimal ++ Set(BooleanType)).foreach { - t => - t match { - case BooleanType => - checkEvaluation(cast(cast(true, BooleanType), t), true) - checkEvaluation(cast(cast(false, BooleanType), t), false) - case _ => - checkEvaluation(cast(cast(true, BooleanType), t), 1) - checkEvaluation(cast(cast(false, BooleanType), t), 0) - } - } - } - - testGluten("missing cases - from byte") { - DataTypeTestUtils.numericTypeWithoutDecimal.foreach { - t => - checkEvaluation(cast(cast(0, ByteType), t), 0) - checkEvaluation(cast(cast(-1, ByteType), t), -1) - checkEvaluation(cast(cast(1, ByteType), t), 1) - } - } - - testGluten("missing cases - from short") { - DataTypeTestUtils.numericTypeWithoutDecimal.foreach { - t => - checkEvaluation(cast(cast(0, ShortType), t), 0) - checkEvaluation(cast(cast(-1, ShortType), t), -1) - checkEvaluation(cast(cast(1, ShortType), t), 1) - } - } - - testGluten("missing cases - date self check") { - val d = Date.valueOf("1970-01-01") - checkEvaluation(cast(d, DateType), d) - } - - testGluten("data type casting") { - val sd = "1970-01-01" - val d = Date.valueOf(sd) - val zts = sd + " 00:00:00" - val sts = sd + " 00:00:02" - val nts = sts + ".1" - val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts)) - - // SystemV timezones are a legacy way of specifying timezones in Unix-like OS. - // It is not supported by Velox. - for (tz <- ALL_TIMEZONES.filterNot(_.getId.contains("SystemV"))) { - withSQLConf( - SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz.getId - ) { - val timeZoneId = Option(tz.getId) - var c = Calendar.getInstance(TimeZoneUTC) - c.set(2015, 2, 8, 2, 30, 0) - checkEvaluation( - cast( - cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId), - TimestampType, - timeZoneId), - millisToMicros(c.getTimeInMillis)) - c = Calendar.getInstance(TimeZoneUTC) - c.set(2015, 10, 1, 2, 30, 0) - checkEvaluation( - cast( - cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId), - TimestampType, - timeZoneId), - millisToMicros(c.getTimeInMillis)) - } - } - - checkEvaluation(cast("abdef", StringType), "abdef") - checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65)) - - checkEvaluation(cast(cast(sd, DateType), StringType), sd) - checkEvaluation(cast(cast(d, StringType), DateType), 0) - - withSQLConf( - SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get - ) { - checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType, UTC_OPT), nts) - checkEvaluation( - cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT), - fromJavaTimestamp(ts)) - - // all convert to string type to check - checkEvaluation( - cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT), StringType), - sd) - checkEvaluation( - cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT), StringType, UTC_OPT), - zts) - } - - checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef") - - checkEvaluation( - cast( - cast(cast(cast(cast(cast("5", ByteType), ShortType), IntegerType), FloatType), DoubleType), - LongType), - 5.toLong) - - checkEvaluation(cast("23", DoubleType), 23d) - checkEvaluation(cast("23", IntegerType), 23) - checkEvaluation(cast("23", FloatType), 23f) - checkEvaluation(cast("23", DecimalType.USER_DEFAULT), Decimal(23)) - checkEvaluation(cast("23", ByteType), 23.toByte) - checkEvaluation(cast("23", ShortType), 23.toShort) - checkEvaluation(cast(123, IntegerType), 123) - - checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null) - } - - test("cast from boolean to timestamp") { - val tsTrue = new Timestamp(0) - tsTrue.setNanos(1000) - - val tsFalse = new Timestamp(0) - - checkEvaluation(cast(true, TimestampType), tsTrue) - - checkEvaluation(cast(false, TimestampType), tsFalse) - } - - testGluten("cast string to timestamp") { - DebuggableThreadUtils.parmap( - ALL_TIMEZONES - .filterNot(_.getId.contains("SystemV")) - .filterNot(_.getId.contains("Europe/Kyiv")) - .filterNot(_.getId.contains("America/Ciudad_Juarez")) - .filterNot(_.getId.contains("Antarctica/Vostok")) - .filterNot(_.getId.contains("Pacific/Kanton")) - .filterNot(_.getId.contains("Asia/Tehran")) - .filterNot(_.getId.contains("Iran")), - prefix = "CastSuiteBase-cast-string-to-timestamp", - maxThreads = 1 - ) { - zid => - withSQLConf( - SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId - ) { - def checkCastStringToTimestamp(str: String, expected: Timestamp): Unit = { - checkEvaluation(cast(Literal(str), TimestampType, Option(zid.getId)), expected) - } - - val tz = TimeZone.getTimeZone(zid) - var c = Calendar.getInstance(tz) - c.set(2015, 0, 1, 0, 0, 0) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015", new Timestamp(c.getTimeInMillis)) - c = Calendar.getInstance(tz) - c.set(2015, 2, 1, 0, 0, 0) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03", new Timestamp(c.getTimeInMillis)) - c = Calendar.getInstance(tz) - c.set(2015, 2, 18, 0, 0, 0) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18 ", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(tz) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18 12:03:17", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18T12:03:17", new Timestamp(c.getTimeInMillis)) - - // If the string value includes timezone string, it represents the timestamp string - // in the timezone regardless of the timeZoneId parameter. - c = Calendar.getInstance(TimeZone.getTimeZone(UTC)) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18T12:03:17Z", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18 12:03:17Z", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17-1:0", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18T12:03:17-01:00", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18T12:03:17+07:30", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17+7:3", - // new Timestamp(c.getTimeInMillis)) - - // tests for the string including milliseconds. - c = Calendar.getInstance(tz) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - checkCastStringToTimestamp("2015-03-18 12:03:17.123", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18T12:03:17.123", new Timestamp(c.getTimeInMillis)) - - // If the string value includes timezone string, it represents the timestamp string - // in the timezone regardless of the timeZoneId parameter. - c = Calendar.getInstance(TimeZone.getTimeZone(UTC)) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 456) - checkCastStringToTimestamp("2015-03-18T12:03:17.456Z", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18 12:03:17.456Z", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17.123-1:0", - // new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp( - "2015-03-18T12:03:17.123-01:00", - new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - checkCastStringToTimestamp( - "2015-03-18T12:03:17.123+07:30", - new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17.123+7:3", - // new Timestamp(c.getTimeInMillis)) - } - } - } - - testGluten("cast decimal to timestamp") { - val tz = TimeZone.getTimeZone(TimeZone.getDefault.getID) - val c = Calendar.getInstance(tz) - c.set(2015, 0, 1, 0, 0, 0) - c.set(Calendar.MILLISECOND, 123) - val d = Decimal(c.getTimeInMillis.toDouble / 1000) - checkEvaluation(cast(d, TimestampType), new Timestamp(c.getTimeInMillis)) - } } diff --git a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala index df973d55c0..46383ebf0f 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/gluten/utils/velox/VeloxTestSettings.scala @@ -107,16 +107,6 @@ class VeloxTestSettings extends BackendTestSettings { enableSuite[GlutenBindReferencesSuite] enableSuite[GlutenBitwiseExpressionsSuite] enableSuite[GlutenCastWithAnsiOffSuite] - .exclude( - "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly. - ) - // Set timezone through config. - .exclude("data type casting") - // Revised by setting timezone through config and commented unsupported cases. - .exclude("cast string to timestamp") - .exclude("cast from timestamp II") - .exclude("SPARK-36286: invalid string cast to timestamp") - .exclude("SPARK-39749: cast Decimal to string") enableSuite[GlutenTryCastSuite] .exclude( "Process Infinity, -Infinity, NaN in case insensitive manner" // +inf not supported in folly. diff --git a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala index efde3f31e0..1e2986c004 100644 --- a/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala +++ b/gluten-ut/spark41/src/test/scala/org/apache/spark/sql/catalyst/expressions/GlutenCastWithAnsiOffSuite.scala @@ -16,15 +16,8 @@ */ package org.apache.spark.sql.catalyst.expressions -import org.apache.spark.sql.GlutenTestsTrait -import org.apache.spark.sql.catalyst.util.DateTimeTestUtils.{withDefaultTimeZone, ALL_TIMEZONES, UTC, UTC_OPT} -import org.apache.spark.sql.catalyst.util.DateTimeUtils.{fromJavaTimestamp, millisToMicros, TimeZoneUTC} import org.apache.spark.sql.internal.SQLConf -import org.apache.spark.sql.types._ -import org.apache.spark.util.DebuggableThreadUtils - -import java.sql.{Date, Timestamp} -import java.util.{Calendar, TimeZone} +import org.apache.spark.sql.shim.GlutenTestsTrait class GlutenCastWithAnsiOffSuite extends CastWithAnsiOffSuite with GlutenTestsTrait { override def beforeAll(): Unit = { @@ -35,269 +28,4 @@ class GlutenCastWithAnsiOffSuite extends CastWithAnsiOffSuite with GlutenTestsTr // In Vanilla spark, the checkEvaluation method doesn't invoke RowEncoder. conf.setConf(SQLConf.PRESERVE_CHAR_VARCHAR_TYPE_INFO, true) } - - override def cast(v: Any, targetType: DataType, timeZoneId: Option[String] = None): Cast = { - v match { - case lit: Expression => - logDebug(s"Cast from: ${lit.dataType.typeName}, to: ${targetType.typeName}") - Cast(lit, targetType, timeZoneId) - case _ => - val lit = Literal(v) - logDebug(s"Cast from: ${lit.dataType.typeName}, to: ${targetType.typeName}") - Cast(lit, targetType, timeZoneId) - } - } - - // Register UDT For test("SPARK-32828") - UDTRegistration.register(classOf[IExampleBaseType].getName, classOf[ExampleBaseTypeUDT].getName) - UDTRegistration.register(classOf[IExampleSubType].getName, classOf[ExampleSubTypeUDT].getName) - - testGluten("missing cases - from boolean") { - (DataTypeTestUtils.numericTypeWithoutDecimal ++ Set(BooleanType)).foreach { - t => - t match { - case BooleanType => - checkEvaluation(cast(cast(true, BooleanType), t), true) - checkEvaluation(cast(cast(false, BooleanType), t), false) - case _ => - checkEvaluation(cast(cast(true, BooleanType), t), 1) - checkEvaluation(cast(cast(false, BooleanType), t), 0) - } - } - } - - testGluten("missing cases - from byte") { - DataTypeTestUtils.numericTypeWithoutDecimal.foreach { - t => - checkEvaluation(cast(cast(0, ByteType), t), 0) - checkEvaluation(cast(cast(-1, ByteType), t), -1) - checkEvaluation(cast(cast(1, ByteType), t), 1) - } - } - - testGluten("missing cases - from short") { - DataTypeTestUtils.numericTypeWithoutDecimal.foreach { - t => - checkEvaluation(cast(cast(0, ShortType), t), 0) - checkEvaluation(cast(cast(-1, ShortType), t), -1) - checkEvaluation(cast(cast(1, ShortType), t), 1) - } - } - - testGluten("missing cases - date self check") { - val d = Date.valueOf("1970-01-01") - checkEvaluation(cast(d, DateType), d) - } - - testGluten("data type casting") { - val sd = "1970-01-01" - val d = Date.valueOf(sd) - val zts = sd + " 00:00:00" - val sts = sd + " 00:00:02" - val nts = sts + ".1" - val ts = withDefaultTimeZone(UTC)(Timestamp.valueOf(nts)) - - // SystemV timezones are a legacy way of specifying timezones in Unix-like OS. - // It is not supported by Velox. - for ( - tz <- ALL_TIMEZONES - .filterNot(_.getId.contains("SystemV")) - .filterNot(_.getId.contains("America/Coyhaique")) - ) { - withSQLConf( - SQLConf.SESSION_LOCAL_TIMEZONE.key -> tz.getId - ) { - val timeZoneId = Option(tz.getId) - var c = Calendar.getInstance(TimeZoneUTC) - c.set(2015, 2, 8, 2, 30, 0) - checkEvaluation( - cast( - cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId), - TimestampType, - timeZoneId), - millisToMicros(c.getTimeInMillis)) - c = Calendar.getInstance(TimeZoneUTC) - c.set(2015, 10, 1, 2, 30, 0) - checkEvaluation( - cast( - cast(new Timestamp(c.getTimeInMillis), StringType, timeZoneId), - TimestampType, - timeZoneId), - millisToMicros(c.getTimeInMillis)) - } - } - - checkEvaluation(cast("abdef", StringType), "abdef") - checkEvaluation(cast("12.65", DecimalType.SYSTEM_DEFAULT), Decimal(12.65)) - - checkEvaluation(cast(cast(sd, DateType), StringType), sd) - checkEvaluation(cast(cast(d, StringType), DateType), 0) - - withSQLConf( - SQLConf.SESSION_LOCAL_TIMEZONE.key -> UTC_OPT.get - ) { - checkEvaluation(cast(cast(nts, TimestampType, UTC_OPT), StringType, UTC_OPT), nts) - checkEvaluation( - cast(cast(ts, StringType, UTC_OPT), TimestampType, UTC_OPT), - fromJavaTimestamp(ts)) - - // all convert to string type to check - checkEvaluation( - cast(cast(cast(nts, TimestampType, UTC_OPT), DateType, UTC_OPT), StringType), - sd) - checkEvaluation( - cast(cast(cast(ts, DateType, UTC_OPT), TimestampType, UTC_OPT), StringType, UTC_OPT), - zts) - } - - checkEvaluation(cast(cast("abdef", BinaryType), StringType), "abdef") - - checkEvaluation( - cast( - cast(cast(cast(cast(cast("5", ByteType), ShortType), IntegerType), FloatType), DoubleType), - LongType), - 5.toLong) - - checkEvaluation(cast("23", DoubleType), 23d) - checkEvaluation(cast("23", IntegerType), 23) - checkEvaluation(cast("23", FloatType), 23f) - checkEvaluation(cast("23", DecimalType.USER_DEFAULT), Decimal(23)) - checkEvaluation(cast("23", ByteType), 23.toByte) - checkEvaluation(cast("23", ShortType), 23.toShort) - checkEvaluation(cast(123, IntegerType), 123) - - checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null) - } - - test("cast from boolean to timestamp") { - val tsTrue = new Timestamp(0) - tsTrue.setNanos(1000) - - val tsFalse = new Timestamp(0) - - checkEvaluation(cast(true, TimestampType), tsTrue) - - checkEvaluation(cast(false, TimestampType), tsFalse) - } - - testGluten("cast string to timestamp") { - DebuggableThreadUtils.parmap( - ALL_TIMEZONES - .filterNot(_.getId.contains("SystemV")) - .filterNot(_.getId.contains("Europe/Kyiv")) - .filterNot(_.getId.contains("America/Ciudad_Juarez")) - .filterNot(_.getId.contains("Antarctica/Vostok")) - .filterNot(_.getId.contains("Pacific/Kanton")) - .filterNot(_.getId.contains("America/Coyhaique")) - .filterNot(_.getId.contains("Asia/Tehran")) - .filterNot(_.getId.contains("Iran")), - prefix = "CastSuiteBase-cast-string-to-timestamp", - maxThreads = 1 - ) { - zid => - withSQLConf( - SQLConf.SESSION_LOCAL_TIMEZONE.key -> zid.getId - ) { - def checkCastStringToTimestamp(str: String, expected: Timestamp): Unit = { - checkEvaluation(cast(Literal(str), TimestampType, Option(zid.getId)), expected) - } - - val tz = TimeZone.getTimeZone(zid) - var c = Calendar.getInstance(tz) - c.set(2015, 0, 1, 0, 0, 0) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015", new Timestamp(c.getTimeInMillis)) - c = Calendar.getInstance(tz) - c.set(2015, 2, 1, 0, 0, 0) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03", new Timestamp(c.getTimeInMillis)) - c = Calendar.getInstance(tz) - c.set(2015, 2, 18, 0, 0, 0) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18 ", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(tz) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18 12:03:17", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18T12:03:17", new Timestamp(c.getTimeInMillis)) - - // If the string value includes timezone string, it represents the timestamp string - // in the timezone regardless of the timeZoneId parameter. - c = Calendar.getInstance(TimeZone.getTimeZone(UTC)) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18T12:03:17Z", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18 12:03:17Z", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17-1:0", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18T12:03:17-01:00", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - checkCastStringToTimestamp("2015-03-18T12:03:17+07:30", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 0) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17+7:3", - // new Timestamp(c.getTimeInMillis)) - - // tests for the string including milliseconds. - c = Calendar.getInstance(tz) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - checkCastStringToTimestamp("2015-03-18 12:03:17.123", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18T12:03:17.123", new Timestamp(c.getTimeInMillis)) - - // If the string value includes timezone string, it represents the timestamp string - // in the timezone regardless of the timeZoneId parameter. - c = Calendar.getInstance(TimeZone.getTimeZone(UTC)) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 456) - checkCastStringToTimestamp("2015-03-18T12:03:17.456Z", new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp("2015-03-18 12:03:17.456Z", new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT-01:00")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17.123-1:0", - // new Timestamp(c.getTimeInMillis)) - checkCastStringToTimestamp( - "2015-03-18T12:03:17.123-01:00", - new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:30")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - checkCastStringToTimestamp( - "2015-03-18T12:03:17.123+07:30", - new Timestamp(c.getTimeInMillis)) - - c = Calendar.getInstance(TimeZone.getTimeZone("GMT+07:03")) - c.set(2015, 2, 18, 12, 3, 17) - c.set(Calendar.MILLISECOND, 123) - // Unsupported timezone format for Velox backend. - // checkCastStringToTimestamp("2015-03-18T12:03:17.123+7:3", - // new Timestamp(c.getTimeInMillis)) - } - } - } - - testGluten("cast decimal to timestamp") { - val tz = TimeZone.getTimeZone(TimeZone.getDefault.getID) - val c = Calendar.getInstance(tz) - c.set(2015, 0, 1, 0, 0, 0) - c.set(Calendar.MILLISECOND, 123) - val d = Decimal(c.getTimeInMillis.toDouble / 1000) - checkEvaluation(cast(d, TimestampType), new Timestamp(c.getTimeInMillis)) - } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
