spark git commit: [SPARK-16287][SQL] Implement str_to_map SQL function

2016-07-21 Thread wenchen 
Repository: spark
Updated Branches:
  refs/heads/master 46f80a307 -> df2c6d59d


[SPARK-16287][SQL] Implement str_to_map SQL function

## What changes were proposed in this pull request?
This PR adds `str_to_map` SQL function in order to remove Hive fallback.

## How was this patch tested?
Pass the Jenkins tests with newly added.

Author: Sandeep Singh 

Closes #13990 from techaddict/SPARK-16287.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/df2c6d59
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/df2c6d59
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/df2c6d59

Branch: refs/heads/master
Commit: df2c6d59d0e1a3db9942dbc5e4993cf3babc2d60
Parents: 46f80a3
Author: Sandeep Singh 
Authored: Fri Jul 22 10:05:21 2016 +0800
Committer: Wenchen Fan 
Committed: Fri Jul 22 10:05:21 2016 +0800

--
 .../catalyst/analysis/FunctionRegistry.scala|  1 +
 .../expressions/complexTypeCreator.scala| 52 +++-
 .../catalyst/expressions/ComplexTypeSuite.scala | 36 ++
 .../apache/spark/sql/StringFunctionsSuite.scala | 23 +
 .../spark/sql/hive/HiveSessionCatalog.scala |  3 +-
 5 files changed, 112 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/df2c6d59/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 65a90d8..6516899 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -228,6 +228,7 @@ object FunctionRegistry {
 expression[Signum]("signum"),
 expression[Sin]("sin"),
 expression[Sinh]("sinh"),
+expression[StringToMap]("str_to_map"),
 expression[Sqrt]("sqrt"),
 expression[Tan]("tan"),
 expression[Tanh]("tanh"),

http://git-wip-us.apache.org/repos/asf/spark/blob/df2c6d59/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index d603d3c..b3c5c58 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions.codegen._
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, 
GenericArrayData, TypeUtils}
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, 
GenericArrayData, MapData, TypeUtils}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -393,3 +393,53 @@ case class CreateNamedStructUnsafe(children: 
Seq[Expression]) extends Expression
 
   override def prettyName: String = "named_struct_unsafe"
 }
+
+/**
+ * Creates a map after splitting the input text into key/value pairs using 
delimiters
+ */
+@ExpressionDescription(
+  usage = "_FUNC_(text[, pairDelim, keyValueDelim]) - Creates a map after 
splitting the text " +
+"into key/value pairs using delimiters. " +
+"Default delimiters are ',' for pairDelim and ':' for keyValueDelim.",
+  extended = """ > SELECT _FUNC_('a:1,b:2,c:3',',',':');\n 
map("a":"1","b":"2","c":"3") """)
+case class StringToMap(text: Expression, pairDelim: Expression, keyValueDelim: 
Expression)
+  extends TernaryExpression with CodegenFallback with ExpectsInputTypes {
+
+  def this(child: Expression, pairDelim: Expression) = {
+this(child, pairDelim, Literal(":"))
+  }
+
+  def this(child: Expression) = {
+this(child, Literal(","), Literal(":"))
+  }
+
+  override def children: Seq[Expression] = Seq(text, pairDelim, keyValueDelim)
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType, 
StringType)
+
+  override def dataType: DataType = MapType(StringType, StringType, 
valueContainsNull = false)
+
+  override def checkInputDataTypes(): TypeCheckResult = {
+if (Seq(pairDelim, keyValueDelim).exists(! _.foldable)) {
+  TypeCheckResult.TypeCheckFailure(s"$prettyName's delimiters must be 
foldable.")
+} else {
+  super.

spark git commit: [SPARK-16287][SQL] Implement str_to_map SQL function

2016-07-21 Thread wenchen 
Repository: spark
Updated Branches:
  refs/heads/branch-2.0 4cb8ff73f -> 70bf8ce72


[SPARK-16287][SQL] Implement str_to_map SQL function

## What changes were proposed in this pull request?
This PR adds `str_to_map` SQL function in order to remove Hive fallback.

## How was this patch tested?
Pass the Jenkins tests with newly added.

Author: Sandeep Singh 

Closes #13990 from techaddict/SPARK-16287.

(cherry picked from commit df2c6d59d0e1a3db9942dbc5e4993cf3babc2d60)
Signed-off-by: Wenchen Fan 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/70bf8ce7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/70bf8ce7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/70bf8ce7

Branch: refs/heads/branch-2.0
Commit: 70bf8ce72a31f7c8b290abea5207244eb4b1fbab
Parents: 4cb8ff7
Author: Sandeep Singh 
Authored: Fri Jul 22 10:05:21 2016 +0800
Committer: Wenchen Fan 
Committed: Fri Jul 22 10:05:41 2016 +0800

--
 .../catalyst/analysis/FunctionRegistry.scala|  1 +
 .../expressions/complexTypeCreator.scala| 52 +++-
 .../catalyst/expressions/ComplexTypeSuite.scala | 36 ++
 .../apache/spark/sql/StringFunctionsSuite.scala | 23 +
 .../spark/sql/hive/HiveSessionCatalog.scala |  3 +-
 5 files changed, 112 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/70bf8ce7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
index 65a90d8..6516899 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
@@ -228,6 +228,7 @@ object FunctionRegistry {
 expression[Signum]("signum"),
 expression[Sin]("sin"),
 expression[Sinh]("sinh"),
+expression[StringToMap]("str_to_map"),
 expression[Sqrt]("sqrt"),
 expression[Tan]("tan"),
 expression[Tanh]("tanh"),

http://git-wip-us.apache.org/repos/asf/spark/blob/70bf8ce7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
index d603d3c..b3c5c58 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult
 import org.apache.spark.sql.catalyst.expressions.codegen._
-import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, 
GenericArrayData, TypeUtils}
+import org.apache.spark.sql.catalyst.util.{ArrayBasedMapData, 
GenericArrayData, MapData, TypeUtils}
 import org.apache.spark.sql.types._
 import org.apache.spark.unsafe.types.UTF8String
 
@@ -393,3 +393,53 @@ case class CreateNamedStructUnsafe(children: 
Seq[Expression]) extends Expression
 
   override def prettyName: String = "named_struct_unsafe"
 }
+
+/**
+ * Creates a map after splitting the input text into key/value pairs using 
delimiters
+ */
+@ExpressionDescription(
+  usage = "_FUNC_(text[, pairDelim, keyValueDelim]) - Creates a map after 
splitting the text " +
+"into key/value pairs using delimiters. " +
+"Default delimiters are ',' for pairDelim and ':' for keyValueDelim.",
+  extended = """ > SELECT _FUNC_('a:1,b:2,c:3',',',':');\n 
map("a":"1","b":"2","c":"3") """)
+case class StringToMap(text: Expression, pairDelim: Expression, keyValueDelim: 
Expression)
+  extends TernaryExpression with CodegenFallback with ExpectsInputTypes {
+
+  def this(child: Expression, pairDelim: Expression) = {
+this(child, pairDelim, Literal(":"))
+  }
+
+  def this(child: Expression) = {
+this(child, Literal(","), Literal(":"))
+  }
+
+  override def children: Seq[Expression] = Seq(text, pairDelim, keyValueDelim)
+
+  override def inputTypes: Seq[AbstractDataType] = Seq(StringType, StringType, 
StringType)
+
+  override def dataType: DataType = MapType(StringType, StringType, 
valueContainsNull = false)
+
+  override def checkInputDataTypes(): TypeCheckResult = {
+if (Seq(pairDelim, keyValueDelim).exists(! _.foldable)) {
+  Ty