spark git commit: [SPARK-23593][SQL] Add interpreted execution for InitializeJavaBean expression

[hvanhovell]

Repository: spark
Updated Branches:
  refs/heads/master b2329fb1f -> d9ca1c906


[SPARK-23593][SQL] Add interpreted execution for InitializeJavaBean expression

## What changes were proposed in this pull request?

Add interpreted execution for `InitializeJavaBean` expression.

## How was this patch tested?

Added unit test.

Author: Liang-Chi Hsieh 

Closes #20985 from viirya/SPARK-23593-2.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d9ca1c90
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d9ca1c90
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d9ca1c90

Branch: refs/heads/master
Commit: d9ca1c906bd0571802f2297c36b407e660fcdb64
Parents: b2329fb
Author: Liang-Chi Hsieh 
Authored: Thu Apr 5 20:43:05 2018 +0200
Committer: Herman van Hovell 
Committed: Thu Apr 5 20:43:05 2018 +0200

--
 .../catalyst/expressions/objects/objects.scala  | 45 --
 .../expressions/ExpressionEvalHelper.scala  |  9 ++--
 .../expressions/ObjectExpressionsSuite.scala| 48 
 3 files changed, 96 insertions(+), 6 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/d9ca1c90/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index 3fa91bd..9252425 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -1420,8 +1420,45 @@ case class InitializeJavaBean(beanInstance: Expression, 
setters: Map[String, Exp
   override def children: Seq[Expression] = beanInstance +: setters.values.toSeq
   override def dataType: DataType = beanInstance.dataType
 
-  override def eval(input: InternalRow): Any =
-throw new UnsupportedOperationException("Only code-generated evaluation is 
supported.")
+  private lazy val resolvedSetters = {
+assert(beanInstance.dataType.isInstanceOf[ObjectType])
+
+val ObjectType(beanClass) = beanInstance.dataType
+setters.map {
+  case (name, expr) =>
+// Looking for known type mapping.
+// But also looking for general `Object`-type parameter for generic 
methods.
+val paramTypes = ScalaReflection.expressionJavaClasses(Seq(expr)) ++ 
Seq(classOf[Object])
+val methods = paramTypes.flatMap { fieldClass =>
+  try {
+Some(beanClass.getDeclaredMethod(name, fieldClass))
+  } catch {
+case e: NoSuchMethodException => None
+  }
+}
+if (methods.isEmpty) {
+  throw new NoSuchMethodException(s"""A method named "$name" is not 
declared """ +
+"in any enclosing class nor any supertype")
+}
+methods.head -> expr
+}
+  }
+
+  override def eval(input: InternalRow): Any = {
+val instance = beanInstance.eval(input)
+if (instance != null) {
+  val bean = instance.asInstanceOf[Object]
+  resolvedSetters.foreach {
+case (setter, expr) =>
+  val paramVal = expr.eval(input)
+  // We don't call setter if input value is null.
+  if (paramVal != null) {
+setter.invoke(bean, paramVal.asInstanceOf[AnyRef])
+  }
+  }
+}
+instance
+  }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
 val instanceGen = beanInstance.genCode(ctx)
@@ -1434,7 +1471,9 @@ case class InitializeJavaBean(beanInstance: Expression, 
setters: Map[String, Exp
 val fieldGen = fieldValue.genCode(ctx)
 s"""
|${fieldGen.code}
-   |$javaBeanInstance.$setterMethod(${fieldGen.value});
+   |if (!${fieldGen.isNull}) {
+   |  $javaBeanInstance.$setterMethod(${fieldGen.value});
+   |}
  """.stripMargin
 }
 val initializeCode = ctx.splitExpressionsWithCurrentInputs(

http://git-wip-us.apache.org/repos/asf/spark/blob/d9ca1c90/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
--
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
index 3828f17..a5ecd1b 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
+++ 

spark git commit: [SPARK-23593][SQL] Add interpreted execution for InitializeJavaBean expression

[hvanhovell]

Repository: spark
Updated Branches:
  refs/heads/master d3bd0435e -> c5c8b5440


[SPARK-23593][SQL] Add interpreted execution for InitializeJavaBean expression

## What changes were proposed in this pull request?

Add interpreted execution for `InitializeJavaBean` expression.

## How was this patch tested?

Added unit test.

Author: Liang-Chi Hsieh 

Closes #20756 from viirya/SPARK-23593.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c5c8b544
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c5c8b544
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c5c8b544

Branch: refs/heads/master
Commit: c5c8b544047a83cb6128a20d31f1d943a15f9260
Parents: d3bd043
Author: Liang-Chi Hsieh 
Authored: Thu Apr 5 13:39:45 2018 +0200
Committer: Herman van Hovell 
Committed: Thu Apr 5 13:39:45 2018 +0200

--
 .../catalyst/expressions/objects/objects.scala  | 47 +-
 .../expressions/ExpressionEvalHelper.scala  |  9 ++--
 .../expressions/ObjectExpressionsSuite.scala| 52 
 3 files changed, 103 insertions(+), 5 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/c5c8b544/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
index a455c1c..20c4f4c 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects/objects.scala
@@ -1410,8 +1410,47 @@ case class InitializeJavaBean(beanInstance: Expression, 
setters: Map[String, Exp
   override def children: Seq[Expression] = beanInstance +: setters.values.toSeq
   override def dataType: DataType = beanInstance.dataType
 
-  override def eval(input: InternalRow): Any =
-throw new UnsupportedOperationException("Only code-generated evaluation is 
supported.")
+  private lazy val resolvedSetters = {
+assert(beanInstance.dataType.isInstanceOf[ObjectType])
+
+val ObjectType(beanClass) = beanInstance.dataType
+setters.map {
+  case (name, expr) =>
+// Looking for known type mapping.
+// But also looking for general `Object`-type parameter for generic 
methods.
+val paramTypes = ScalaReflection.expressionJavaClasses(Seq(expr)) ++ 
Seq(classOf[Object])
+val methods = paramTypes.flatMap { fieldClass =>
+  try {
+Some(beanClass.getDeclaredMethod(name, fieldClass))
+  } catch {
+case e: NoSuchMethodException => None
+  }
+}
+if (methods.isEmpty) {
+  throw new NoSuchMethodException(s"""A method named "$name" is not 
declared """ +
+"in any enclosing class nor any supertype")
+}
+methods.head -> expr
+}
+  }
+
+  override def eval(input: InternalRow): Any = {
+val instance = beanInstance.eval(input)
+if (instance != null) {
+  val bean = instance.asInstanceOf[Object]
+  resolvedSetters.foreach {
+case (setter, expr) =>
+  val paramVal = expr.eval(input)
+  if (paramVal == null) {
+throw new NullPointerException("The parameter value for setters in 
" +
+  "`InitializeJavaBean` can not be null")
+  } else {
+setter.invoke(bean, paramVal.asInstanceOf[AnyRef])
+  }
+  }
+}
+instance
+  }
 
   override def doGenCode(ctx: CodegenContext, ev: ExprCode): ExprCode = {
 val instanceGen = beanInstance.genCode(ctx)
@@ -1424,6 +1463,10 @@ case class InitializeJavaBean(beanInstance: Expression, 
setters: Map[String, Exp
 val fieldGen = fieldValue.genCode(ctx)
 s"""
|${fieldGen.code}
+   |if (${fieldGen.isNull}) {
+   |  throw new NullPointerException("The parameter value for setters 
in " +
+   |"`InitializeJavaBean` can not be null");
+   |}
|$javaBeanInstance.$setterMethod(${fieldGen.value});
  """.stripMargin
 }

http://git-wip-us.apache.org/repos/asf/spark/blob/c5c8b544/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
--
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ExpressionEvalHelper.scala
index 3828f17..a5ecd1b 100644
---