git commit: [SPARK-2052] [SQL] Add optimization for CaseConversionExpression's.

2014-06-11 Thread marmbrus 
Repository: spark
Updated Branches:
  refs/heads/branch-1.0 81049eb54 -> e3955643d


[SPARK-2052] [SQL] Add optimization for CaseConversionExpression's.

Add optimization for `CaseConversionExpression`'s.

Author: Takuya UESHIN 

Closes #990 from ueshin/issues/SPARK-2052 and squashes the following commits:

2568666 [Takuya UESHIN] Move some rules back.
dde7ede [Takuya UESHIN] Add tests to check if ConstantFolding can handle null 
literals and remove the unneeded rules from NullPropagation.
c4eea67 [Takuya UESHIN] Fix toString methods.
23e2363 [Takuya UESHIN] Make CaseConversionExpressions foldable if the child is 
foldable.
0ff7568 [Takuya UESHIN] Add tests for collapsing case statements.
3977d80 [Takuya UESHIN] Add optimization for CaseConversionExpression's.

(cherry picked from commit 9a2448daf984d5bb550dfe0d9e28cbb80ef5cb51)
Signed-off-by: Michael Armbrust 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e3955643
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e3955643
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e3955643

Branch: refs/heads/branch-1.0
Commit: e3955643d6f838146e8b2e0463b27612d8e48d02
Parents: 81049eb
Author: Takuya UESHIN 
Authored: Wed Jun 11 17:58:35 2014 -0700
Committer: Michael Armbrust 
Committed: Wed Jun 11 17:58:44 2014 -0700

--
 .../catalyst/expressions/stringOperations.scala |  7 +-
 .../sql/catalyst/optimizer/Optimizer.scala  | 30 ---
 .../optimizer/ConstantFoldingSuite.scala| 61 -
 ...SimplifyCaseConversionExpressionsSuite.scala | 91 
 4 files changed, 174 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/e3955643/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
index 4203034..c074b7b 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
@@ -76,7 +76,8 @@ trait CaseConversionExpression {
   type EvaluatedType = Any
 
   def convert(v: String): String
-  
+
+  override def foldable: Boolean = child.foldable
   def nullable: Boolean = child.nullable
   def dataType: DataType = StringType
 
@@ -142,6 +143,8 @@ case class RLike(left: Expression, right: Expression)
 case class Upper(child: Expression) extends UnaryExpression with 
CaseConversionExpression {
   
   override def convert(v: String): String = v.toUpperCase()
+
+  override def toString() = s"Upper($child)"
 }
 
 /**
@@ -150,4 +153,6 @@ case class Upper(child: Expression) extends UnaryExpression 
with CaseConversionE
 case class Lower(child: Expression) extends UnaryExpression with 
CaseConversionExpression {
   
   override def convert(v: String): String = v.toLowerCase()
+
+  override def toString() = s"Lower($child)"
 }

http://git-wip-us.apache.org/repos/asf/spark/blob/e3955643/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 28d1aa2..25a347b 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -36,7 +36,8 @@ object Optimizer extends RuleExecutor[LogicalPlan] {
   ConstantFolding,
   BooleanSimplification,
   SimplifyFilters,
-  SimplifyCasts) ::
+  SimplifyCasts,
+  SimplifyCaseConversionExpressions) ::
 Batch("Filter Pushdown", FixedPoint(100),
   CombineFilters,
   PushPredicateThroughProject,
@@ -132,18 +133,6 @@ object NullPropagation extends Rule[LogicalPlan] {
   case Literal(candidate, _) if candidate == v => true
   case _ => false
 })) => Literal(true, BooleanType)
-  case e: UnaryMinus => e.child match {
-case Literal(null, _) => Literal(null, e.dataType)
-case _ => e
-  }
-  case e: Cast => e.child match {
-case Literal(null, _) => Literal(null, e.dataType)
-case _ => e
-  }
-  case e: Not => e.child match {
-case Literal(null, _) => Literal(null, e.dataType)
-case _ => e
-  }
   // Put exceptional cases above if any
   case e: BinaryArithmetic => e.children match {

git commit: [SPARK-2052] [SQL] Add optimization for CaseConversionExpression's.

2014-06-11 Thread marmbrus 
Repository: spark
Updated Branches:
  refs/heads/master d45e0c6b9 -> 9a2448daf


[SPARK-2052] [SQL] Add optimization for CaseConversionExpression's.

Add optimization for `CaseConversionExpression`'s.

Author: Takuya UESHIN 

Closes #990 from ueshin/issues/SPARK-2052 and squashes the following commits:

2568666 [Takuya UESHIN] Move some rules back.
dde7ede [Takuya UESHIN] Add tests to check if ConstantFolding can handle null 
literals and remove the unneeded rules from NullPropagation.
c4eea67 [Takuya UESHIN] Fix toString methods.
23e2363 [Takuya UESHIN] Make CaseConversionExpressions foldable if the child is 
foldable.
0ff7568 [Takuya UESHIN] Add tests for collapsing case statements.
3977d80 [Takuya UESHIN] Add optimization for CaseConversionExpression's.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9a2448da
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9a2448da
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9a2448da

Branch: refs/heads/master
Commit: 9a2448daf984d5bb550dfe0d9e28cbb80ef5cb51
Parents: d45e0c6
Author: Takuya UESHIN 
Authored: Wed Jun 11 17:58:35 2014 -0700
Committer: Michael Armbrust 
Committed: Wed Jun 11 17:58:35 2014 -0700

--
 .../catalyst/expressions/stringOperations.scala |  7 +-
 .../sql/catalyst/optimizer/Optimizer.scala  | 30 ---
 .../optimizer/ConstantFoldingSuite.scala| 61 -
 ...SimplifyCaseConversionExpressionsSuite.scala | 91 
 4 files changed, 174 insertions(+), 15 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/9a2448da/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
index 4203034..c074b7b 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/stringOperations.scala
@@ -76,7 +76,8 @@ trait CaseConversionExpression {
   type EvaluatedType = Any
 
   def convert(v: String): String
-  
+
+  override def foldable: Boolean = child.foldable
   def nullable: Boolean = child.nullable
   def dataType: DataType = StringType
 
@@ -142,6 +143,8 @@ case class RLike(left: Expression, right: Expression)
 case class Upper(child: Expression) extends UnaryExpression with 
CaseConversionExpression {
   
   override def convert(v: String): String = v.toUpperCase()
+
+  override def toString() = s"Upper($child)"
 }
 
 /**
@@ -150,4 +153,6 @@ case class Upper(child: Expression) extends UnaryExpression 
with CaseConversionE
 case class Lower(child: Expression) extends UnaryExpression with 
CaseConversionExpression {
   
   override def convert(v: String): String = v.toLowerCase()
+
+  override def toString() = s"Lower($child)"
 }

http://git-wip-us.apache.org/repos/asf/spark/blob/9a2448da/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
index 28d1aa2..25a347b 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/optimizer/Optimizer.scala
@@ -36,7 +36,8 @@ object Optimizer extends RuleExecutor[LogicalPlan] {
   ConstantFolding,
   BooleanSimplification,
   SimplifyFilters,
-  SimplifyCasts) ::
+  SimplifyCasts,
+  SimplifyCaseConversionExpressions) ::
 Batch("Filter Pushdown", FixedPoint(100),
   CombineFilters,
   PushPredicateThroughProject,
@@ -132,18 +133,6 @@ object NullPropagation extends Rule[LogicalPlan] {
   case Literal(candidate, _) if candidate == v => true
   case _ => false
 })) => Literal(true, BooleanType)
-  case e: UnaryMinus => e.child match {
-case Literal(null, _) => Literal(null, e.dataType)
-case _ => e
-  }
-  case e: Cast => e.child match {
-case Literal(null, _) => Literal(null, e.dataType)
-case _ => e
-  }
-  case e: Not => e.child match {
-case Literal(null, _) => Literal(null, e.dataType)
-case _ => e
-  }
   // Put exceptional cases above if any
   case e: BinaryArithmetic => e.children match {
 case Literal(null, _) :: right :: Nil => Literal(null, e.dataType)
@@ -375,3 +364,18 @@ object CombineL