This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 71882f1 [SPARK-28343][FOLLOW-UP][SQL][TEST] Enable
spark.sql.function.preferIntegralDivision for PostgreSQL testing
71882f1 is described below
commit 71882f119e72934a02d4c177f0d52c785e2df79f
Author: Yuming Wang <yumw...@ebay.com>
AuthorDate: Tue Jul 16 08:46:01 2019 -0700
[SPARK-28343][FOLLOW-UP][SQL][TEST] Enable
spark.sql.function.preferIntegralDivision for PostgreSQL testing
## What changes were proposed in this pull request?
This PR enables `spark.sql.function.preferIntegralDivision` for PostgreSQL
testing.
## How was this patch tested?
N/A
Closes #25170 from wangyum/SPARK-28343-2.
Authored-by: Yuming Wang <yumw...@ebay.com>
Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
.../test/resources/sql-tests/inputs/pgSQL/int2.sql | 6 +-
.../test/resources/sql-tests/inputs/pgSQL/int4.sql | 1 -
.../test/resources/sql-tests/inputs/pgSQL/int8.sql | 1 -
.../resources/sql-tests/results/pgSQL/case.sql.out | 18 ++---
.../resources/sql-tests/results/pgSQL/int2.sql.out | 4 +-
.../resources/sql-tests/results/pgSQL/int4.sql.out | 32 ++++-----
.../resources/sql-tests/results/pgSQL/int8.sql.out | 78 +++++++++++-----------
.../sql-tests/results/udf/pgSQL/udf-case.sql.out | 8 +--
.../org/apache/spark/sql/SQLQueryTestSuite.scala | 1 +
9 files changed, 73 insertions(+), 76 deletions(-)
diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/int2.sql
b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/int2.sql
index 61f350d..f64ec5d 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/int2.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/int2.sql
@@ -88,11 +88,9 @@ WHERE f1 > -32767;
SELECT '' AS five, i.f1, i.f1 - int('2') AS x FROM INT2_TBL i;
--- PostgreSQL `/` is the same with Spark `div` since SPARK-2659.
-SELECT '' AS five, i.f1, i.f1 div smallint('2') AS x FROM INT2_TBL i;
+SELECT '' AS five, i.f1, i.f1 / smallint('2') AS x FROM INT2_TBL i;
--- PostgreSQL `/` is the same with Spark `div` since SPARK-2659.
-SELECT '' AS five, i.f1, i.f1 div int('2') AS x FROM INT2_TBL i;
+SELECT '' AS five, i.f1, i.f1 / int('2') AS x FROM INT2_TBL i;
-- corner cases
SELECT string(shiftleft(smallint(-1), 15));
diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/int4.sql
b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/int4.sql
index 675636e..86432a8 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/int4.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/int4.sql
@@ -135,7 +135,6 @@ SELECT int('1000') < int('999') AS `false`;
SELECT 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 AS ten;
--- [SPARK-2659] HiveQL: Division operator should always perform fractional
division
SELECT 2 + 2 / 2 AS three;
SELECT (2 + 2) / 2 AS two;
diff --git a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/int8.sql
b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/int8.sql
index 32ac877..d29bf3b 100644
--- a/sql/core/src/test/resources/sql-tests/inputs/pgSQL/int8.sql
+++ b/sql/core/src/test/resources/sql-tests/inputs/pgSQL/int8.sql
@@ -85,7 +85,6 @@ SELECT 37 - q1 AS minus4 FROM INT8_TBL;
SELECT '' AS five, 2 * q1 AS `twice int4` FROM INT8_TBL;
SELECT '' AS five, q1 * 2 AS `twice int4` FROM INT8_TBL;
--- [SPARK-2659] HiveQL: Division operator should always perform fractional
division
-- int8 op int4
SELECT q1 + int(42) AS `8plus4`, q1 - int(42) AS `8minus4`, q1 * int(42) AS
`8mul4`, q1 / int(42) AS `8div4` FROM INT8_TBL;
-- int4 op int8
diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/case.sql.out
b/sql/core/src/test/resources/sql-tests/results/pgSQL/case.sql.out
index 9b20b31..f95adcd 100644
--- a/sql/core/src/test/resources/sql-tests/results/pgSQL/case.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/case.sql.out
@@ -176,28 +176,28 @@ struct<None:string,NULL on no matches:int>
-- !query 18
SELECT CASE WHEN 1=0 THEN 1/0 WHEN 1=1 THEN 1 ELSE 2/0 END
-- !query 18 schema
-struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) WHEN (1
= 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE))
END:double>
+struct<CASE WHEN (1 = 0) THEN (1 div 0) WHEN (1 = 1) THEN 1 ELSE (2 div 0)
END:int>
-- !query 18 output
-1.0
+1
-- !query 19
SELECT CASE 1 WHEN 0 THEN 1/0 WHEN 1 THEN 1 ELSE 2/0 END
-- !query 19 schema
-struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) WHEN (1
= 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) / CAST(0 AS DOUBLE))
END:double>
+struct<CASE WHEN (1 = 0) THEN (1 div 0) WHEN (1 = 1) THEN 1 ELSE (2 div 0)
END:int>
-- !query 19 output
-1.0
+1
-- !query 20
SELECT CASE WHEN i > 100 THEN 1/0 ELSE 0 END FROM case_tbl
-- !query 20 schema
-struct<CASE WHEN (i > 100) THEN (CAST(1 AS DOUBLE) / CAST(0 AS DOUBLE)) ELSE
CAST(0 AS DOUBLE) END:double>
+struct<CASE WHEN (i > 100) THEN (1 div 0) ELSE 0 END:int>
-- !query 20 output
-0.0
-0.0
-0.0
-0.0
+0
+0
+0
+0
-- !query 21
diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/int2.sql.out
b/sql/core/src/test/resources/sql-tests/results/pgSQL/int2.sql.out
index 6b9246f..7a7ce5f 100644
--- a/sql/core/src/test/resources/sql-tests/results/pgSQL/int2.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/int2.sql.out
@@ -266,7 +266,7 @@ struct<five:string,f1:smallint,x:int>
-- !query 27
-SELECT '' AS five, i.f1, i.f1 div smallint('2') AS x FROM INT2_TBL i
+SELECT '' AS five, i.f1, i.f1 / smallint('2') AS x FROM INT2_TBL i
-- !query 27 schema
struct<five:string,f1:smallint,x:smallint>
-- !query 27 output
@@ -278,7 +278,7 @@ struct<five:string,f1:smallint,x:smallint>
-- !query 28
-SELECT '' AS five, i.f1, i.f1 div int('2') AS x FROM INT2_TBL i
+SELECT '' AS five, i.f1, i.f1 / int('2') AS x FROM INT2_TBL i
-- !query 28 schema
struct<five:string,f1:smallint,x:int>
-- !query 28 output
diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/int4.sql.out
b/sql/core/src/test/resources/sql-tests/results/pgSQL/int4.sql.out
index 08fb4d4..456b1ef 100644
--- a/sql/core/src/test/resources/sql-tests/results/pgSQL/int4.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/int4.sql.out
@@ -339,25 +339,25 @@ struct<five:string,f1:int,x:int>
-- !query 33
SELECT '' AS five, i.f1, i.f1 / smallint('2') AS x FROM INT4_TBL i
-- !query 33 schema
-struct<five:string,f1:int,x:double>
+struct<five:string,f1:int,x:int>
-- !query 33 output
--123456 -61728.0
- -2147483647 -1.0737418235E9
- 0 0.0
- 123456 61728.0
- 2147483647 1.0737418235E9
+-123456 -61728
+ -2147483647 -1073741823
+ 0 0
+ 123456 61728
+ 2147483647 1073741823
-- !query 34
SELECT '' AS five, i.f1, i.f1 / int('2') AS x FROM INT4_TBL i
-- !query 34 schema
-struct<five:string,f1:int,x:double>
+struct<five:string,f1:int,x:int>
-- !query 34 output
--123456 -61728.0
- -2147483647 -1.0737418235E9
- 0 0.0
- 123456 61728.0
- 2147483647 1.0737418235E9
+-123456 -61728
+ -2147483647 -1073741823
+ 0 0
+ 123456 61728
+ 2147483647 1073741823
-- !query 35
@@ -435,17 +435,17 @@ struct<ten:int>
-- !query 44
SELECT 2 + 2 / 2 AS three
-- !query 44 schema
-struct<three:double>
+struct<three:int>
-- !query 44 output
-3.0
+3
-- !query 45
SELECT (2 + 2) / 2 AS two
-- !query 45 schema
-struct<two:double>
+struct<two:int>
-- !query 45 output
-2.0
+2
-- !query 46
diff --git a/sql/core/src/test/resources/sql-tests/results/pgSQL/int8.sql.out
b/sql/core/src/test/resources/sql-tests/results/pgSQL/int8.sql.out
index 880b257..6d7fae1 100644
--- a/sql/core/src/test/resources/sql-tests/results/pgSQL/int8.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/pgSQL/int8.sql.out
@@ -412,13 +412,13 @@ struct<three:string,q1:bigint,q2:bigint,multiply:bigint>
-- !query 42
SELECT '' AS five, q1, q2, q1 / q2 AS divide, q1 % q2 AS mod FROM INT8_TBL
-- !query 42 schema
-struct<five:string,q1:bigint,q2:bigint,divide:double,mod:bigint>
+struct<five:string,q1:bigint,q2:bigint,divide:bigint,mod:bigint>
-- !query 42 output
-123 456 0.26973684210526316 123
- 123 4567890123456789 2.6927092525360204E-14 123
- 4567890123456789 -4567890123456789 -1.0 0
- 4567890123456789 123 3.713731807688446E13 57
- 4567890123456789 4567890123456789 1.0 0
+123 456 0 123
+ 123 4567890123456789 0 123
+ 4567890123456789 -4567890123456789 -1 0
+ 4567890123456789 123 37137318076884 57
+ 4567890123456789 4567890123456789 1 0
-- !query 43
@@ -496,49 +496,49 @@ struct<five:string,twice int4:bigint>
-- !query 49
SELECT q1 + int(42) AS `8plus4`, q1 - int(42) AS `8minus4`, q1 * int(42) AS
`8mul4`, q1 / int(42) AS `8div4` FROM INT8_TBL
-- !query 49 schema
-struct<8plus4:bigint,8minus4:bigint,8mul4:bigint,8div4:double>
+struct<8plus4:bigint,8minus4:bigint,8mul4:bigint,8div4:bigint>
-- !query 49 output
-165 81 5166 2.9285714285714284
-165 81 5166 2.9285714285714284
-4567890123456831 4567890123456747 191851385185185138
1.0875928865373308E14
-4567890123456831 4567890123456747 191851385185185138
1.0875928865373308E14
-4567890123456831 4567890123456747 191851385185185138
1.0875928865373308E14
+165 81 5166 2
+165 81 5166 2
+4567890123456831 4567890123456747 191851385185185138
108759288653733
+4567890123456831 4567890123456747 191851385185185138
108759288653733
+4567890123456831 4567890123456747 191851385185185138
108759288653733
-- !query 50
SELECT int(246) + q1 AS `4plus8`, int(246) - q1 AS `4minus8`, int(246) * q1 AS
`4mul8`, int(246) / q1 AS `4div8` FROM INT8_TBL
-- !query 50 schema
-struct<4plus8:bigint,4minus8:bigint,4mul8:bigint,4div8:double>
+struct<4plus8:bigint,4minus8:bigint,4mul8:bigint,4div8:bigint>
-- !query 50 output
-369 123 30258 2.0
-369 123 30258 2.0
-4567890123457035 -4567890123456543 1123700970370370094
5.385418505072041E-14
-4567890123457035 -4567890123456543 1123700970370370094
5.385418505072041E-14
-4567890123457035 -4567890123456543 1123700970370370094
5.385418505072041E-14
+369 123 30258 2
+369 123 30258 2
+4567890123457035 -4567890123456543 1123700970370370094 0
+4567890123457035 -4567890123456543 1123700970370370094 0
+4567890123457035 -4567890123456543 1123700970370370094 0
-- !query 51
SELECT q1 + smallint(42) AS `8plus2`, q1 - smallint(42) AS `8minus2`, q1 *
smallint(42) AS `8mul2`, q1 / smallint(42) AS `8div2` FROM INT8_TBL
-- !query 51 schema
-struct<8plus2:bigint,8minus2:bigint,8mul2:bigint,8div2:double>
+struct<8plus2:bigint,8minus2:bigint,8mul2:bigint,8div2:bigint>
-- !query 51 output
-165 81 5166 2.9285714285714284
-165 81 5166 2.9285714285714284
-4567890123456831 4567890123456747 191851385185185138
1.0875928865373308E14
-4567890123456831 4567890123456747 191851385185185138
1.0875928865373308E14
-4567890123456831 4567890123456747 191851385185185138
1.0875928865373308E14
+165 81 5166 2
+165 81 5166 2
+4567890123456831 4567890123456747 191851385185185138
108759288653733
+4567890123456831 4567890123456747 191851385185185138
108759288653733
+4567890123456831 4567890123456747 191851385185185138
108759288653733
-- !query 52
SELECT smallint(246) + q1 AS `2plus8`, smallint(246) - q1 AS `2minus8`,
smallint(246) * q1 AS `2mul8`, smallint(246) / q1 AS `2div8` FROM INT8_TBL
-- !query 52 schema
-struct<2plus8:bigint,2minus8:bigint,2mul8:bigint,2div8:double>
+struct<2plus8:bigint,2minus8:bigint,2mul8:bigint,2div8:bigint>
-- !query 52 output
-369 123 30258 2.0
-369 123 30258 2.0
-4567890123457035 -4567890123456543 1123700970370370094
5.385418505072041E-14
-4567890123457035 -4567890123456543 1123700970370370094
5.385418505072041E-14
-4567890123457035 -4567890123456543 1123700970370370094
5.385418505072041E-14
+369 123 30258 2
+369 123 30258 2
+4567890123457035 -4567890123456543 1123700970370370094 0
+4567890123457035 -4567890123456543 1123700970370370094 0
+4567890123457035 -4567890123456543 1123700970370370094 0
-- !query 53
@@ -572,7 +572,7 @@ struct<max(q1):bigint,max(q2):bigint>
-- !query 56
select bigint('9223372036854775800') / bigint('0')
-- !query 56 schema
-struct<(CAST(CAST(9223372036854775800 AS BIGINT) AS DOUBLE) / CAST(CAST(0 AS
BIGINT) AS DOUBLE)):double>
+struct<(CAST(9223372036854775800 AS BIGINT) div CAST(0 AS BIGINT)):bigint>
-- !query 56 output
NULL
@@ -580,7 +580,7 @@ NULL
-- !query 57
select bigint('-9223372036854775808') / smallint('0')
-- !query 57 schema
-struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(0 AS
SMALLINT) AS DOUBLE)):double>
+struct<(CAST(-9223372036854775808 AS BIGINT) div CAST(CAST(0 AS SMALLINT) AS
BIGINT)):bigint>
-- !query 57 output
NULL
@@ -588,7 +588,7 @@ NULL
-- !query 58
select smallint('100') / bigint('0')
-- !query 58 schema
-struct<(CAST(CAST(100 AS SMALLINT) AS DOUBLE) / CAST(CAST(0 AS BIGINT) AS
DOUBLE)):double>
+struct<(CAST(CAST(100 AS SMALLINT) AS BIGINT) div CAST(0 AS BIGINT)):bigint>
-- !query 58 output
NULL
@@ -744,9 +744,9 @@ struct<(CAST(-9223372036854775808 AS BIGINT) * CAST(-1 AS
BIGINT)):bigint>
-- !query 74
SELECT bigint((-9223372036854775808)) / bigint((-1))
-- !query 74 schema
-struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(-1 AS
BIGINT) AS DOUBLE)):double>
+struct<(CAST(-9223372036854775808 AS BIGINT) div CAST(-1 AS BIGINT)):bigint>
-- !query 74 output
-9.223372036854776E18
+-9223372036854775808
-- !query 75
@@ -768,9 +768,9 @@ struct<(CAST(-9223372036854775808 AS BIGINT) * CAST(CAST(-1
AS INT) AS BIGINT)):
-- !query 77
SELECT bigint((-9223372036854775808)) / int((-1))
-- !query 77 schema
-struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(-1 AS
INT) AS DOUBLE)):double>
+struct<(CAST(-9223372036854775808 AS BIGINT) div CAST(CAST(-1 AS INT) AS
BIGINT)):bigint>
-- !query 77 output
-9.223372036854776E18
+-9223372036854775808
-- !query 78
@@ -792,9 +792,9 @@ struct<(CAST(-9223372036854775808 AS BIGINT) * CAST(CAST(-1
AS SMALLINT) AS BIGI
-- !query 80
SELECT bigint((-9223372036854775808)) / smallint((-1))
-- !query 80 schema
-struct<(CAST(CAST(-9223372036854775808 AS BIGINT) AS DOUBLE) / CAST(CAST(-1 AS
SMALLINT) AS DOUBLE)):double>
+struct<(CAST(-9223372036854775808 AS BIGINT) div CAST(CAST(-1 AS SMALLINT) AS
BIGINT)):bigint>
-- !query 80 output
-9.223372036854776E18
+-9223372036854775808
-- !query 81
diff --git
a/sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-case.sql.out
b/sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-case.sql.out
index 6bb7a78..91650e1 100644
--- a/sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-case.sql.out
+++ b/sql/core/src/test/resources/sql-tests/results/udf/pgSQL/udf-case.sql.out
@@ -176,15 +176,15 @@ struct<None:string,NULL on no matches:int>
-- !query 18
SELECT CASE WHEN CAST(udf(1=0) AS boolean) THEN 1/0 WHEN 1=1 THEN 1 ELSE 2/0
END
-- !query 18 schema
-struct<CASE WHEN CAST(udf((1 = 0)) AS BOOLEAN) THEN (CAST(1 AS DOUBLE) /
CAST(0 AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE)
/ CAST(0 AS DOUBLE)) END:double>
+struct<CASE WHEN CAST(udf((1 = 0)) AS BOOLEAN) THEN (1 div 0) WHEN (1 = 1)
THEN 1 ELSE (2 div 0) END:int>
-- !query 18 output
-1.0
+1
-- !query 19
SELECT CASE 1 WHEN 0 THEN 1/udf(0) WHEN 1 THEN 1 ELSE 2/0 END
-- !query 19 schema
-struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(CAST(udf(0) AS DOUBLE)
AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE (CAST(2 AS DOUBLE) /
CAST(0 AS DOUBLE)) END:double>
+struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) / CAST(CAST(udf(0) AS DOUBLE)
AS DOUBLE)) WHEN (1 = 1) THEN CAST(1 AS DOUBLE) ELSE CAST((2 div 0) AS DOUBLE)
END:double>
-- !query 19 output
1.0
@@ -192,7 +192,7 @@ struct<CASE WHEN (1 = 0) THEN (CAST(1 AS DOUBLE) /
CAST(CAST(udf(0) AS DOUBLE) A
-- !query 20
SELECT CASE WHEN i > 100 THEN udf(1/0) ELSE udf(0) END FROM case_tbl
-- !query 20 schema
-struct<CASE WHEN (i > 100) THEN udf((cast(1 as double) / cast(0 as double)))
ELSE udf(0) END:string>
+struct<CASE WHEN (i > 100) THEN udf((1 div 0)) ELSE udf(0) END:string>
-- !query 20 output
0
0
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
index 1085f03..e4052b7 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/SQLQueryTestSuite.scala
@@ -293,6 +293,7 @@ class SQLQueryTestSuite extends QueryTest with
SharedSQLContext {
// PostgreSQL enabled cartesian product by default.
localSparkSession.conf.set(SQLConf.CROSS_JOINS_ENABLED.key, true)
localSparkSession.conf.set(SQLConf.ANSI_SQL_PARSER.key, true)
+ localSparkSession.conf.set(SQLConf.PREFER_INTEGRAL_DIVISION.key, true)
case _ =>
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
