Repository: spark Updated Branches: refs/heads/master 833c8d415 -> a2b3b6762
[SPARK-19089][SQL] Add support for nested sequences ## What changes were proposed in this pull request? Replaced specific sequence encoders with generic sequence encoder to enable nesting of sequences. Does not add support for nested arrays as that cannot be solved in this way. ## How was this patch tested? ```bash build/mvn -DskipTests clean package && dev/run-tests ``` Additionally in Spark shell: ``` scala> Seq(Seq(Seq(1))).toDS.collect() res0: Array[Seq[Seq[Int]]] = Array(List(List(1))) ``` Author: Michal Senkyr <mike.sen...@gmail.com> Closes #18011 from michalsenkyr/dataset-seq-nested. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/a2b3b676 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/a2b3b676 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/a2b3b676 Branch: refs/heads/master Commit: a2b3b67624ce7bbb29ddade03c1791d95e51869b Parents: 833c8d4 Author: Michal Senkyr <mike.sen...@gmail.com> Authored: Mon May 22 16:49:19 2017 +0800 Committer: Wenchen Fan <wenc...@databricks.com> Committed: Mon May 22 16:49:19 2017 +0800 ---------------------------------------------------------------------- .../org/apache/spark/sql/SQLImplicits.scala | 55 ++++---------------- .../spark/sql/DatasetPrimitiveSuite.scala | 5 ++ 2 files changed, 16 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/a2b3b676/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala index 375df64..17671ea 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala @@ -111,93 +111,60 @@ abstract class SQLImplicits extends LowPrioritySQLImplicits { /** * @since 1.6.1 - * @deprecated use [[newIntSequenceEncoder]] + * @deprecated use [[newSequenceEncoder]] */ def newIntSeqEncoder: Encoder[Seq[Int]] = ExpressionEncoder() /** * @since 1.6.1 - * @deprecated use [[newLongSequenceEncoder]] + * @deprecated use [[newSequenceEncoder]] */ def newLongSeqEncoder: Encoder[Seq[Long]] = ExpressionEncoder() /** * @since 1.6.1 - * @deprecated use [[newDoubleSequenceEncoder]] + * @deprecated use [[newSequenceEncoder]] */ def newDoubleSeqEncoder: Encoder[Seq[Double]] = ExpressionEncoder() /** * @since 1.6.1 - * @deprecated use [[newFloatSequenceEncoder]] + * @deprecated use [[newSequenceEncoder]] */ def newFloatSeqEncoder: Encoder[Seq[Float]] = ExpressionEncoder() /** * @since 1.6.1 - * @deprecated use [[newByteSequenceEncoder]] + * @deprecated use [[newSequenceEncoder]] */ def newByteSeqEncoder: Encoder[Seq[Byte]] = ExpressionEncoder() /** * @since 1.6.1 - * @deprecated use [[newShortSequenceEncoder]] + * @deprecated use [[newSequenceEncoder]] */ def newShortSeqEncoder: Encoder[Seq[Short]] = ExpressionEncoder() /** * @since 1.6.1 - * @deprecated use [[newBooleanSequenceEncoder]] + * @deprecated use [[newSequenceEncoder]] */ def newBooleanSeqEncoder: Encoder[Seq[Boolean]] = ExpressionEncoder() /** * @since 1.6.1 - * @deprecated use [[newStringSequenceEncoder]] + * @deprecated use [[newSequenceEncoder]] */ def newStringSeqEncoder: Encoder[Seq[String]] = ExpressionEncoder() /** * @since 1.6.1 - * @deprecated use [[newProductSequenceEncoder]] + * @deprecated use [[newSequenceEncoder]] */ - implicit def newProductSeqEncoder[A <: Product : TypeTag]: Encoder[Seq[A]] = ExpressionEncoder() + def newProductSeqEncoder[A <: Product : TypeTag]: Encoder[Seq[A]] = ExpressionEncoder() /** @since 2.2.0 */ - implicit def newIntSequenceEncoder[T <: Seq[Int] : TypeTag]: Encoder[T] = - ExpressionEncoder() - - /** @since 2.2.0 */ - implicit def newLongSequenceEncoder[T <: Seq[Long] : TypeTag]: Encoder[T] = - ExpressionEncoder() - - /** @since 2.2.0 */ - implicit def newDoubleSequenceEncoder[T <: Seq[Double] : TypeTag]: Encoder[T] = - ExpressionEncoder() - - /** @since 2.2.0 */ - implicit def newFloatSequenceEncoder[T <: Seq[Float] : TypeTag]: Encoder[T] = - ExpressionEncoder() - - /** @since 2.2.0 */ - implicit def newByteSequenceEncoder[T <: Seq[Byte] : TypeTag]: Encoder[T] = - ExpressionEncoder() - - /** @since 2.2.0 */ - implicit def newShortSequenceEncoder[T <: Seq[Short] : TypeTag]: Encoder[T] = - ExpressionEncoder() - - /** @since 2.2.0 */ - implicit def newBooleanSequenceEncoder[T <: Seq[Boolean] : TypeTag]: Encoder[T] = - ExpressionEncoder() - - /** @since 2.2.0 */ - implicit def newStringSequenceEncoder[T <: Seq[String] : TypeTag]: Encoder[T] = - ExpressionEncoder() - - /** @since 2.2.0 */ - implicit def newProductSequenceEncoder[T <: Seq[Product] : TypeTag]: Encoder[T] = - ExpressionEncoder() + implicit def newSequenceEncoder[T <: Seq[_] : TypeTag]: Encoder[T] = ExpressionEncoder() // Arrays http://git-wip-us.apache.org/repos/asf/spark/blob/a2b3b676/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala index 5415653..7e2949a 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetPrimitiveSuite.scala @@ -258,6 +258,11 @@ class DatasetPrimitiveSuite extends QueryTest with SharedSQLContext { ListClass(List(1)) -> Queue("test" -> SeqClass(Seq(2)))) } + test("nested sequences") { + checkDataset(Seq(Seq(Seq(1))).toDS(), Seq(Seq(1))) + checkDataset(Seq(List(Queue(1))).toDS(), List(Queue(1))) + } + test("package objects") { import packageobject._ checkDataset(Seq(PackageClass(1)).toDS(), PackageClass(1)) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
