spark git commit: [SPARK-16729][SQL] Throw analysis exception for invalid date casts

2016-07-27 Thread rxin 
Repository: spark
Updated Branches:
  refs/heads/branch-2.0 4e98e6905 -> 8bc2877d8


[SPARK-16729][SQL] Throw analysis exception for invalid date casts

Spark currently throws exceptions for invalid casts for all other data types 
except date type. Somehow date type returns null. It should be consistent and 
throws analysis exception as well.

Added a unit test case in CastSuite.

Author: petermaxlee 

Closes #14358 from petermaxlee/SPARK-16729.

(cherry picked from commit ef0ccbcb07252db0ead8509e70d1a9a670d41616)
Signed-off-by: Reynold Xin 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/8bc2877d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/8bc2877d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/8bc2877d

Branch: refs/heads/branch-2.0
Commit: 8bc2877d8c7cad6831de73a3f7c032b7dd73ae78
Parents: 4e98e69
Author: petermaxlee 
Authored: Wed Jul 27 16:04:43 2016 +0800
Committer: Reynold Xin 
Committed: Wed Jul 27 08:10:06 2016 -0700

--
 .../spark/sql/catalyst/expressions/Cast.scala   |  9 ++---
 .../spark/sql/catalyst/expressions/CastSuite.scala  | 16 ++--
 2 files changed, 16 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/8bc2877d/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index b1e89b5..a12fba0 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -52,7 +52,8 @@ object Cast {
 case (DateType, TimestampType) => true
 case (_: NumericType, TimestampType) => true
 
-case (_, DateType) => true
+case (StringType, DateType) => true
+case (TimestampType, DateType) => true
 
 case (StringType, CalendarIntervalType) => true
 
@@ -228,18 +229,12 @@ case class Cast(child: Expression, dataType: DataType) 
extends UnaryExpression w
   // throw valid precision more than seconds, according to Hive.
   // Timestamp.nanos is in 0 to 999,999,999, no more than a second.
   buildCast[Long](_, t => DateTimeUtils.millisToDays(t / 1000L))
-// Hive throws this exception as a Semantic Exception
-// It is never possible to compare result when hive return with exception,
-// so we can return null
-// NULL is more reasonable here, since the query itself obeys the grammar.
-case _ => _ => null
   }
 
   // IntervalConverter
   private[this] def castToInterval(from: DataType): Any => Any = from match {
 case StringType =>
   buildCast[UTF8String](_, s => CalendarInterval.fromString(s.toString))
-case _ => _ => null
   }
 
   // LongConverter

http://git-wip-us.apache.org/repos/asf/spark/blob/8bc2877d/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
--
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index dfda7c5..5ae0527 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -70,7 +70,8 @@ class CastSuite extends SparkFunSuite with 
ExpressionEvalHelper {
 checkNullCast(DateType, TimestampType)
 numericTypes.foreach(dt => checkNullCast(dt, TimestampType))
 
-atomicTypes.foreach(dt => checkNullCast(dt, DateType))
+checkNullCast(StringType, DateType)
+checkNullCast(TimestampType, DateType)
 
 checkNullCast(StringType, CalendarIntervalType)
 numericTypes.foreach(dt => checkNullCast(StringType, dt))
@@ -366,7 +367,6 @@ class CastSuite extends SparkFunSuite with 
ExpressionEvalHelper {
 checkEvaluation(cast("2012-12-11", DoubleType), null)
 checkEvaluation(cast(123, IntegerType), 123)
 
-
 checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null)
   }
 
@@ -783,4 +783,16 @@ class CastSuite extends SparkFunSuite with 
ExpressionEvalHelper {
 checkEvaluation(cast("abc", BooleanType), null)
 checkEvaluation(cast("", BooleanType), null)
   }
+
+  test("SPARK-16729 type checking for casting to date type") {
+assert(cast("1234", DateType).checkInputDataTypes().isSuccess)
+assert(cast(new Timestamp(1), DateType).checkInputDataTypes().isSuccess)
+assert(cast(false, DateType).checkInputDataTypes().isFailure)
+assert(cast(1.toByte, Da

spark git commit: [SPARK-16729][SQL] Throw analysis exception for invalid date casts

2016-07-27 Thread wenchen 
Repository: spark
Updated Branches:
  refs/heads/master 5b8e848bb -> ef0ccbcb0


[SPARK-16729][SQL] Throw analysis exception for invalid date casts

## What changes were proposed in this pull request?
Spark currently throws exceptions for invalid casts for all other data types 
except date type. Somehow date type returns null. It should be consistent and 
throws analysis exception as well.

## How was this patch tested?
Added a unit test case in CastSuite.

Author: petermaxlee 

Closes #14358 from petermaxlee/SPARK-16729.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ef0ccbcb
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ef0ccbcb
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ef0ccbcb

Branch: refs/heads/master
Commit: ef0ccbcb07252db0ead8509e70d1a9a670d41616
Parents: 5b8e848
Author: petermaxlee 
Authored: Wed Jul 27 16:04:43 2016 +0800
Committer: Wenchen Fan 
Committed: Wed Jul 27 16:04:43 2016 +0800

--
 .../spark/sql/catalyst/expressions/Cast.scala   |  9 ++---
 .../spark/sql/catalyst/expressions/CastSuite.scala  | 16 ++--
 2 files changed, 16 insertions(+), 9 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/ef0ccbcb/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
index b1e89b5..a12fba0 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala
@@ -52,7 +52,8 @@ object Cast {
 case (DateType, TimestampType) => true
 case (_: NumericType, TimestampType) => true
 
-case (_, DateType) => true
+case (StringType, DateType) => true
+case (TimestampType, DateType) => true
 
 case (StringType, CalendarIntervalType) => true
 
@@ -228,18 +229,12 @@ case class Cast(child: Expression, dataType: DataType) 
extends UnaryExpression w
   // throw valid precision more than seconds, according to Hive.
   // Timestamp.nanos is in 0 to 999,999,999, no more than a second.
   buildCast[Long](_, t => DateTimeUtils.millisToDays(t / 1000L))
-// Hive throws this exception as a Semantic Exception
-// It is never possible to compare result when hive return with exception,
-// so we can return null
-// NULL is more reasonable here, since the query itself obeys the grammar.
-case _ => _ => null
   }
 
   // IntervalConverter
   private[this] def castToInterval(from: DataType): Any => Any = from match {
 case StringType =>
   buildCast[UTF8String](_, s => CalendarInterval.fromString(s.toString))
-case _ => _ => null
   }
 
   // LongConverter

http://git-wip-us.apache.org/repos/asf/spark/blob/ef0ccbcb/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
--
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
index dfda7c5..5ae0527 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuite.scala
@@ -70,7 +70,8 @@ class CastSuite extends SparkFunSuite with 
ExpressionEvalHelper {
 checkNullCast(DateType, TimestampType)
 numericTypes.foreach(dt => checkNullCast(dt, TimestampType))
 
-atomicTypes.foreach(dt => checkNullCast(dt, DateType))
+checkNullCast(StringType, DateType)
+checkNullCast(TimestampType, DateType)
 
 checkNullCast(StringType, CalendarIntervalType)
 numericTypes.foreach(dt => checkNullCast(StringType, dt))
@@ -366,7 +367,6 @@ class CastSuite extends SparkFunSuite with 
ExpressionEvalHelper {
 checkEvaluation(cast("2012-12-11", DoubleType), null)
 checkEvaluation(cast(123, IntegerType), 123)
 
-
 checkEvaluation(cast(Literal.create(null, IntegerType), ShortType), null)
   }
 
@@ -783,4 +783,16 @@ class CastSuite extends SparkFunSuite with 
ExpressionEvalHelper {
 checkEvaluation(cast("abc", BooleanType), null)
 checkEvaluation(cast("", BooleanType), null)
   }
+
+  test("SPARK-16729 type checking for casting to date type") {
+assert(cast("1234", DateType).checkInputDataTypes().isSuccess)
+assert(cast(new Timestamp(1), DateType).checkInputDataTypes().isSuccess)
+assert(cast(false, DateType).checkInputDataTypes().isFailure)
+assert(cast(1.toByte, DateType).checkInputDataTy