spark git commit: [SPARK-11954][SQL] Encoder for JavaBeans
Repository: spark
Updated Branches:
refs/heads/branch-1.6 74a230676 -> 88bbce008
[SPARK-11954][SQL] Encoder for JavaBeans
create java version of `constructorFor` and `extractorFor` in
`JavaTypeInference`
Author: Wenchen Fan
This patch had conflicts when merged, resolved by
Committer: Michael Armbrust
Closes #9937 from cloud-fan/pojo.
(cherry picked from commit fd95eeaf491809c6bb0f83d46b37b5e2eebbcbca)
Signed-off-by: Michael Armbrust
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/88bbce00
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/88bbce00
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/88bbce00
Branch: refs/heads/branch-1.6
Commit: 88bbce00813acf23eef411ac354c35995ddf9e77
Parents: 74a2306
Author: Wenchen Fan
Authored: Tue Dec 1 10:35:12 2015 -0800
Committer: Michael Armbrust
Committed: Tue Dec 1 10:35:26 2015 -0800
--
.../scala/org/apache/spark/sql/Encoder.scala| 18 ++
.../spark/sql/catalyst/JavaTypeInference.scala | 313 ++-
.../catalyst/encoders/ExpressionEncoder.scala | 21 +-
.../sql/catalyst/expressions/objects.scala | 42 ++-
.../spark/sql/catalyst/trees/TreeNode.scala | 27 +-
.../sql/catalyst/util/ArrayBasedMapData.scala | 5 +
.../sql/catalyst/util/GenericArrayData.scala| 3 +
.../sql/catalyst/trees/TreeNodeSuite.scala | 25 ++
.../org/apache/spark/sql/JavaDatasetSuite.java | 174 ++-
9 files changed, 608 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/88bbce00/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
--
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
index 03aa25e..c40061a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
@@ -98,6 +98,24 @@ object Encoders {
def STRING: Encoder[java.lang.String] = ExpressionEncoder()
/**
+ * Creates an encoder for Java Bean of type T.
+ *
+ * T must be publicly accessible.
+ *
+ * supported types for java bean field:
+ * - primitive types: boolean, int, double, etc.
+ * - boxed types: Boolean, Integer, Double, etc.
+ * - String
+ * - java.math.BigDecimal
+ * - time related: java.sql.Date, java.sql.Timestamp
+ * - collection types: only array and java.util.List currently, map support
is in progress
+ * - nested java bean.
+ *
+ * @since 1.6.0
+ */
+ def bean[T](beanClass: Class[T]): Encoder[T] =
ExpressionEncoder.javaBean(beanClass)
+
+ /**
* (Scala-specific) Creates an encoder that serializes objects of type T
using Kryo.
* This encoder maps T into a single byte array (binary) field.
*
http://git-wip-us.apache.org/repos/asf/spark/blob/88bbce00/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
index 7d4cfbe..c8ee87e 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
@@ -17,14 +17,20 @@
package org.apache.spark.sql.catalyst
-import java.beans.Introspector
+import java.beans.{PropertyDescriptor, Introspector}
import java.lang.{Iterable => JIterable}
-import java.util.{Iterator => JIterator, Map => JMap}
+import java.util.{Iterator => JIterator, Map => JMap, List => JList}
import scala.language.existentials
import com.google.common.reflect.TypeToken
+
import org.apache.spark.sql.types._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute,
UnresolvedExtractValue}
+import org.apache.spark.sql.catalyst.util.{GenericArrayData,
ArrayBasedMapData, DateTimeUtils}
+import org.apache.spark.unsafe.types.UTF8String
+
/**
* Type-inference utilities for POJOs and Java collections.
@@ -33,13 +39,14 @@ object JavaTypeInference {
private val iterableType = TypeToken.of(classOf[JIterable[_]])
private val mapType = TypeToken.of(classOf[JMap[_, _]])
+ private val listType = TypeToken.of(classOf[JList[_]])
private val iteratorReturnType =
classOf[JIterable[_]].getMethod("iterator").getGenericReturnType
private val nextReturnType =
classOf[JIterator[_]].getMethod("next").getGenericReturnType
private val keySetReturnType = classOf[JMap[_,
_]].getMethod("keySet").getGenericReturnType
private
spark git commit: [SPARK-11954][SQL] Encoder for JavaBeans
Repository: spark
Updated Branches:
refs/heads/master 9df24624a -> fd95eeaf4
[SPARK-11954][SQL] Encoder for JavaBeans
create java version of `constructorFor` and `extractorFor` in
`JavaTypeInference`
Author: Wenchen Fan
This patch had conflicts when merged, resolved by
Committer: Michael Armbrust
Closes #9937 from cloud-fan/pojo.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/fd95eeaf
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/fd95eeaf
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/fd95eeaf
Branch: refs/heads/master
Commit: fd95eeaf491809c6bb0f83d46b37b5e2eebbcbca
Parents: 9df2462
Author: Wenchen Fan
Authored: Tue Dec 1 10:35:12 2015 -0800
Committer: Michael Armbrust
Committed: Tue Dec 1 10:35:12 2015 -0800
--
.../scala/org/apache/spark/sql/Encoder.scala| 18 ++
.../spark/sql/catalyst/JavaTypeInference.scala | 313 ++-
.../catalyst/encoders/ExpressionEncoder.scala | 21 +-
.../sql/catalyst/expressions/objects.scala | 42 ++-
.../spark/sql/catalyst/trees/TreeNode.scala | 27 +-
.../sql/catalyst/util/ArrayBasedMapData.scala | 5 +
.../sql/catalyst/util/GenericArrayData.scala| 3 +
.../sql/catalyst/trees/TreeNodeSuite.scala | 25 ++
.../org/apache/spark/sql/JavaDatasetSuite.java | 174 ++-
9 files changed, 608 insertions(+), 20 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/fd95eeaf/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
--
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
index 03aa25e..c40061a 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/Encoder.scala
@@ -98,6 +98,24 @@ object Encoders {
def STRING: Encoder[java.lang.String] = ExpressionEncoder()
/**
+ * Creates an encoder for Java Bean of type T.
+ *
+ * T must be publicly accessible.
+ *
+ * supported types for java bean field:
+ * - primitive types: boolean, int, double, etc.
+ * - boxed types: Boolean, Integer, Double, etc.
+ * - String
+ * - java.math.BigDecimal
+ * - time related: java.sql.Date, java.sql.Timestamp
+ * - collection types: only array and java.util.List currently, map support
is in progress
+ * - nested java bean.
+ *
+ * @since 1.6.0
+ */
+ def bean[T](beanClass: Class[T]): Encoder[T] =
ExpressionEncoder.javaBean(beanClass)
+
+ /**
* (Scala-specific) Creates an encoder that serializes objects of type T
using Kryo.
* This encoder maps T into a single byte array (binary) field.
*
http://git-wip-us.apache.org/repos/asf/spark/blob/fd95eeaf/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
--
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
index 7d4cfbe..c8ee87e 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/JavaTypeInference.scala
@@ -17,14 +17,20 @@
package org.apache.spark.sql.catalyst
-import java.beans.Introspector
+import java.beans.{PropertyDescriptor, Introspector}
import java.lang.{Iterable => JIterable}
-import java.util.{Iterator => JIterator, Map => JMap}
+import java.util.{Iterator => JIterator, Map => JMap, List => JList}
import scala.language.existentials
import com.google.common.reflect.TypeToken
+
import org.apache.spark.sql.types._
+import org.apache.spark.sql.catalyst.expressions._
+import org.apache.spark.sql.catalyst.analysis.{UnresolvedAttribute,
UnresolvedExtractValue}
+import org.apache.spark.sql.catalyst.util.{GenericArrayData,
ArrayBasedMapData, DateTimeUtils}
+import org.apache.spark.unsafe.types.UTF8String
+
/**
* Type-inference utilities for POJOs and Java collections.
@@ -33,13 +39,14 @@ object JavaTypeInference {
private val iterableType = TypeToken.of(classOf[JIterable[_]])
private val mapType = TypeToken.of(classOf[JMap[_, _]])
+ private val listType = TypeToken.of(classOf[JList[_]])
private val iteratorReturnType =
classOf[JIterable[_]].getMethod("iterator").getGenericReturnType
private val nextReturnType =
classOf[JIterator[_]].getMethod("next").getGenericReturnType
private val keySetReturnType = classOf[JMap[_,
_]].getMethod("keySet").getGenericReturnType
private val valuesReturnType = classOf[JMap[_,
_]].getMethod("values").getGenericReturnType
/**
- * Infers the
