This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 06ac7d5 [SPARK-27922][SQL][PYTHON][TESTS] Convert and port 'natural-join.sql' into UDF test base 06ac7d5 is described below commit 06ac7d5966a2f072d523de8f798e601473a49871 Author: manu.zhang <manu.zh...@vipshop.com> AuthorDate: Thu Jul 11 09:37:25 2019 +0900 [SPARK-27922][SQL][PYTHON][TESTS] Convert and port 'natural-join.sql' into UDF test base ## What changes were proposed in this pull request? This PR adds some tests converted from `natural-join.sql` to test UDFs following the combination guide in [SPARK-27921](https://issues.apache.org/jira/browse/SPARK-27921). <details><summary>Diff results comparing to `natural-join.sql`</summary> <p> ```diff diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join. sql.out index 43f2f9a..53ef177 100644 --- a/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out -27,7 +27,7 struct<> -- !query 2 -SELECT * FROM nt1 natural join nt2 where k = "one" +SELECT * FROM nt1 natural join nt2 where udf(k) = "one" -- !query 2 schema struct<k:string,v1:int,v2:int> -- !query 2 output -36,7 +36,7 one 1 5 -- !query 3 -SELECT * FROM nt1 natural left join nt2 order by v1, v2 +SELECT * FROM nt1 natural left join nt2 where k <> udf("") order by v1, v2 -- !query 3 schema diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join. sql.out index 43f2f9a..53ef177 100644 --- a/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out +++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out -27,7 +27,7 struct<> -- !query 2 -SELECT * FROM nt1 natural join nt2 where k = "one" +SELECT * FROM nt1 natural join nt2 where udf(k) = "one" -- !query 2 schema struct<k:string,v1:int,v2:int> -- !query 2 output -36,7 +36,7 one 1 5 -- !query 3 -SELECT * FROM nt1 natural left join nt2 order by v1, v2 +SELECT * FROM nt1 natural left join nt2 where k <> udf("") order by v1, v2 -- !query 3 schema struct<k:string,v1:int,v2:int> ``` </p> </details> ## How was this patch tested? Tested as guided in [SPARK-27921](https://issues.apache.org/jira/browse/SPARK-27921). Closes #25088 from manuzhang/SPARK-27922. Authored-by: manu.zhang <manu.zh...@vipshop.com> Signed-off-by: HyukjinKwon <gurwls...@apache.org> --- .../sql-tests/inputs/udf/udf-natural-join.sql | 29 ++++++++++ .../sql-tests/results/udf/udf-natural-join.sql.out | 64 ++++++++++++++++++++++ 2 files changed, 93 insertions(+) diff --git a/sql/core/src/test/resources/sql-tests/inputs/udf/udf-natural-join.sql b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-natural-join.sql new file mode 100644 index 0000000..6862683 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/inputs/udf/udf-natural-join.sql @@ -0,0 +1,29 @@ +-- List of configuration the test suite is run against: +--SET spark.sql.autoBroadcastJoinThreshold=10485760 +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=true +--SET spark.sql.autoBroadcastJoinThreshold=-1,spark.sql.join.preferSortMergeJoin=false + +-- This test file was converted from natural-join.sql. +-- Note that currently registered UDF returns a string. So there are some differences, for instance +-- in string cast within UDF in Scala and Python. + +create temporary view nt1 as select * from values + ("one", 1), + ("two", 2), + ("three", 3) + as nt1(k, v1); + +create temporary view nt2 as select * from values + ("one", 1), + ("two", 22), + ("one", 5) + as nt2(k, v2); + + +SELECT * FROM nt1 natural join nt2 where udf(k) = "one"; + +SELECT * FROM nt1 natural left join nt2 where k <> udf("") order by v1, v2; + +SELECT * FROM nt1 natural right join nt2 where udf(k) <> udf("") order by v1, v2; + +SELECT udf(count(*)) FROM nt1 natural full outer join nt2; diff --git a/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out b/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out new file mode 100644 index 0000000..53ef177 --- /dev/null +++ b/sql/core/src/test/resources/sql-tests/results/udf/udf-natural-join.sql.out @@ -0,0 +1,64 @@ +-- Automatically generated by SQLQueryTestSuite +-- Number of queries: 6 + + +-- !query 0 +create temporary view nt1 as select * from values + ("one", 1), + ("two", 2), + ("three", 3) + as nt1(k, v1) +-- !query 0 schema +struct<> +-- !query 0 output + + + +-- !query 1 +create temporary view nt2 as select * from values + ("one", 1), + ("two", 22), + ("one", 5) + as nt2(k, v2) +-- !query 1 schema +struct<> +-- !query 1 output + + + +-- !query 2 +SELECT * FROM nt1 natural join nt2 where udf(k) = "one" +-- !query 2 schema +struct<k:string,v1:int,v2:int> +-- !query 2 output +one 1 1 +one 1 5 + + +-- !query 3 +SELECT * FROM nt1 natural left join nt2 where k <> udf("") order by v1, v2 +-- !query 3 schema +struct<k:string,v1:int,v2:int> +-- !query 3 output +one 1 1 +one 1 5 +two 2 22 +three 3 NULL + + +-- !query 4 +SELECT * FROM nt1 natural right join nt2 where udf(k) <> udf("") order by v1, v2 +-- !query 4 schema +struct<k:string,v1:int,v2:int> +-- !query 4 output +one 1 1 +one 1 5 +two 2 22 + + +-- !query 5 +SELECT udf(count(*)) FROM nt1 natural full outer join nt2 +-- !query 5 schema +struct<udf(count(1)):string> +-- !query 5 output +4 --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org