This is an automated email from the ASF dual-hosted git repository. wenchen pushed a commit to branch branch-3.0 in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.0 by this push: new 61f5b9a [SPARK-31409][SQL][TEST] Fix failed tests due to result order changing when enable AQE 61f5b9a is described below commit 61f5b9a7c09a89dbacdec067cee4aadef069c4ef Author: yi.wu <yi...@databricks.com> AuthorDate: Mon Apr 13 14:36:25 2020 +0000 [SPARK-31409][SQL][TEST] Fix failed tests due to result order changing when enable AQE ### What changes were proposed in this pull request? This PR fix two tests by avoid result order changing when we enable AQE: 1. In `SQLQueryTestSuite`, disable BHJ optimization to avoid changing result order 2. In test `SQLQuerySuite#check outputs of expression examples`, disable `spark.sql.adaptive.coalescePartitions.enabled` to avoid changing result order ### Why are the changes needed? query 147 in SQLQueryTestSuite#"udf/postgreSQL/udf-join.sql - Scala UDF" and test sql/SQLQuerySuite#"check outputs of expression examples" can fail when enable AQE due to result order changing. And this PR fix them. ### Does this PR introduce any user-facing change? No. ### How was this patch tested? Tested manually with AQE enabled. Closes #28178 from Ngone51/fix_order. Authored-by: yi.wu <yi...@databricks.com> Signed-off-by: Wenchen Fan <wenc...@databricks.com> (cherry picked from commit f6512903da6caef44fd2833327546dd38cc32a59) Signed-off-by: Wenchen Fan <wenc...@databricks.com> --- .../src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql | 3 +++ sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala | 2 ++ 2 files changed, 5 insertions(+) diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql index e6fe107..77bcfe7 100644 --- a/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql +++ b/sql/core/src/test/resources/sql-tests/inputs/udf/postgreSQL/udf-join.sql @@ -8,6 +8,9 @@ -- -- This test file was converted from postgreSQL/join.sql. +-- Disable BroadcastHashJoin optimization to avoid changing result order when we enable AQE +--SET spark.sql.autoBroadcastJoinThreshold = -1 + CREATE OR REPLACE TEMPORARY VIEW INT4_TBL AS SELECT * FROM (VALUES (0), (123456), (-123456), (2147483647), (-2147483647)) AS v(f1); diff --git a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala index 42f0d96..3c514f2 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQuerySuite.scala @@ -181,6 +181,8 @@ class SQLQuerySuite extends QueryTest with SharedSparkSession with AdaptiveSpark parFuncs.foreach { funcId => // Examples can change settings. We clone the session to prevent tests clashing. val clonedSpark = spark.cloneSession() + // Coalescing partitions can change result order, so disable it. + clonedSpark.sessionState.conf.setConf(SQLConf.COALESCE_PARTITIONS_ENABLED, false) val info = clonedSpark.sessionState.catalog.lookupFunctionInfo(funcId) val className = info.getClassName if (!ignoreSet.contains(className)) { --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org