Repository: spark Updated Branches: refs/heads/master 9b23e92c7 -> 2848f4da4
[SPARK-8809][SQL] Remove ConvertNaNs analyzer rule. "NaN" from string to double is already handled by Cast expression itself. Author: Reynold Xin <r...@databricks.com> Closes #7206 from rxin/convertnans and squashes the following commits: 3d99c33 [Reynold Xin] [SPARK-8809][SQL] Remove ConvertNaNs analyzer rule. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2848f4da Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2848f4da Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2848f4da Branch: refs/heads/master Commit: 2848f4da47d5c395de93ab9960bd905edfbd3439 Parents: 9b23e92 Author: Reynold Xin <r...@databricks.com> Authored: Fri Jul 3 00:25:02 2015 -0700 Committer: Reynold Xin <r...@databricks.com> Committed: Fri Jul 3 00:25:02 2015 -0700 ---------------------------------------------------------------------- .../catalyst/analysis/HiveTypeCoercion.scala | 33 -------------------- 1 file changed, 33 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/2848f4da/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala index 6006e7b..38eb832 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/HiveTypeCoercion.scala @@ -35,7 +35,6 @@ object HiveTypeCoercion { val typeCoercionRules = PropagateTypes :: - ConvertNaNs :: InConversion :: WidenTypes :: PromoteStrings :: @@ -149,38 +148,6 @@ object HiveTypeCoercion { } /** - * Converts string "NaN"s that are in binary operators with a NaN-able types (Float / Double) to - * the appropriate numeric equivalent. - */ - // TODO: remove this rule and make Cast handle Nan. - object ConvertNaNs extends Rule[LogicalPlan] { - private val StringNaN = Literal("NaN") - - def apply(plan: LogicalPlan): LogicalPlan = plan transform { - case q: LogicalPlan => q transformExpressions { - // Skip nodes who's children have not been resolved yet. - case e if !e.childrenResolved => e - - /* Double Conversions */ - case b @ BinaryOperator(StringNaN, right @ DoubleType()) => - b.makeCopy(Array(Literal(Double.NaN), right)) - case b @ BinaryOperator(left @ DoubleType(), StringNaN) => - b.makeCopy(Array(left, Literal(Double.NaN))) - - /* Float Conversions */ - case b @ BinaryOperator(StringNaN, right @ FloatType()) => - b.makeCopy(Array(Literal(Float.NaN), right)) - case b @ BinaryOperator(left @ FloatType(), StringNaN) => - b.makeCopy(Array(left, Literal(Float.NaN))) - - /* Use float NaN by default to avoid unnecessary type widening */ - case b @ BinaryOperator(left @ StringNaN, StringNaN) => - b.makeCopy(Array(left, Literal(Float.NaN))) - } - } - } - - /** * Widens numeric types and converts strings to numbers when appropriate. * * Loosely based on rules from "Hadoop: The Definitive Guide" 2nd edition, by Tom White --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
