Repository: spark Updated Branches: refs/heads/master a81ef9e1f -> 9c25d7f73
[SPARK-25431][SQL][EXAMPLES] Fix function examples and unify the format of the example results. ## What changes were proposed in this pull request? There are some mistakes in examples of newly added functions. Also the format of the example results are not unified. We should fix and unify them. ## How was this patch tested? Manually executed the examples. Closes #22421 from ueshin/issues/SPARK-25431/fix_examples. Authored-by: Takuya UESHIN <ues...@databricks.com> Signed-off-by: gatorsmile <gatorsm...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/9c25d7f7 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/9c25d7f7 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/9c25d7f7 Branch: refs/heads/master Commit: 9c25d7f735ed8c49c795babea3fda3cab226e7cb Parents: a81ef9e Author: Takuya UESHIN <ues...@databricks.com> Authored: Fri Sep 14 09:25:27 2018 -0700 Committer: gatorsmile <gatorsm...@gmail.com> Committed: Fri Sep 14 09:25:27 2018 -0700 ---------------------------------------------------------------------- .../expressions/collectionOperations.scala | 49 ++++++++++---------- .../expressions/complexTypeCreator.scala | 4 +- .../expressions/higherOrderFunctions.scala | 32 ++++++------- 3 files changed, 43 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/9c25d7f7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala index ea6fccc..3ad21ec 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/collectionOperations.scala @@ -131,7 +131,7 @@ case class Size(child: Expression) extends UnaryExpression with ExpectsInputType examples = """ Examples: > SELECT _FUNC_(map(1, 'a', 2, 'b')); - [1,2] + [1, 2] """) case class MapKeys(child: Expression) extends UnaryExpression with ExpectsInputTypes { @@ -320,7 +320,7 @@ case class ArraysZip(children: Seq[Expression]) extends Expression with ExpectsI examples = """ Examples: > SELECT _FUNC_(map(1, 'a', 2, 'b')); - ["a","b"] + ["a", "b"] """) case class MapValues(child: Expression) extends UnaryExpression with ExpectsInputTypes { @@ -348,7 +348,7 @@ case class MapValues(child: Expression) examples = """ Examples: > SELECT _FUNC_(map(1, 'a', 2, 'b')); - [(1,"a"),(2,"b")] + [[1, "a"], [2, "b"]] """, since = "2.4.0") case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInputTypes { @@ -516,7 +516,7 @@ case class MapEntries(child: Expression) extends UnaryExpression with ExpectsInp examples = """ Examples: > SELECT _FUNC_(map(1, 'a', 2, 'b'), map(2, 'c', 3, 'd')); - [[1 -> "a"], [2 -> "b"], [2 -> "c"], [3 -> "d"]] + [1 -> "a", 2 -> "b", 2 -> "c", 3 -> "d"] """, since = "2.4.0") case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpression { @@ -718,7 +718,7 @@ case class MapConcat(children: Seq[Expression]) extends ComplexTypeMergingExpres examples = """ Examples: > SELECT _FUNC_(array(struct(1, 'a'), struct(2, 'b'))); - {1:"a",2:"b"} + [1 -> "a", 2 -> "b"] """, since = "2.4.0") case class MapFromEntries(child: Expression) extends UnaryExpression { @@ -1071,7 +1071,7 @@ object ArraySortLike { examples = """ Examples: > SELECT _FUNC_(array('b', 'd', null, 'c', 'a'), true); - [null,"a","b","c","d"] + [null, "a", "b", "c", "d"] """) // scalastyle:on line.size.limit case class SortArray(base: Expression, ascendingOrder: Expression) @@ -1129,7 +1129,7 @@ case class SortArray(base: Expression, ascendingOrder: Expression) examples = """ Examples: > SELECT _FUNC_(array('b', 'd', null, 'c', 'a')); - ["a","b","c","d",null] + ["a", "b", "c", "d", null] """, since = "2.4.0") // scalastyle:on line.size.limit @@ -1254,7 +1254,7 @@ case class Shuffle(child: Expression, randomSeed: Option[Long] = None) examples = """ Examples: > SELECT _FUNC_('Spark SQL'); - LQS krapS + "LQS krapS" > SELECT _FUNC_(array(2, 1, 4, 3)); [3, 4, 1, 2] """, @@ -1634,9 +1634,9 @@ case class ArraysOverlap(left: Expression, right: Expression) examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, 4), 2, 2); - [2,3] + [2, 3] > SELECT _FUNC_(array(1, 2, 3, 4), -2, 2); - [3,4] + [3, 4] """, since = "2.4.0") // scalastyle:on line.size.limit case class Slice(x: Expression, start: Expression, length: Expression) @@ -1745,11 +1745,11 @@ case class Slice(x: Expression, start: Expression, length: Expression) examples = """ Examples: > SELECT _FUNC_(array('hello', 'world'), ' '); - hello world + "hello world" > SELECT _FUNC_(array('hello', null ,'world'), ' '); - hello world + "hello world" > SELECT _FUNC_(array('hello', null ,'world'), ' ', ','); - hello , world + "hello , world" """, since = "2.4.0") case class ArrayJoin( array: Expression, @@ -2236,10 +2236,11 @@ case class ElementAt(left: Expression, right: Expression) extends GetMapValueUti examples = """ Examples: > SELECT _FUNC_('Spark', 'SQL'); - SparkSQL + "SparkSQL" > SELECT _FUNC_(array(1, 2, 3), array(4, 5), array(6)); - | [1,2,3,4,5,6] - """) + [1, 2, 3, 4, 5, 6] + """, + note = "Concat logic for arrays is available since 2.4.0.") case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpression { private def allowedTypes: Seq[AbstractDataType] = Seq(StringType, BinaryType, ArrayType) @@ -2427,8 +2428,8 @@ case class Concat(children: Seq[Expression]) extends ComplexTypeMergingExpressio usage = "_FUNC_(arrayOfArrays) - Transforms an array of arrays into a single array.", examples = """ Examples: - > SELECT _FUNC_(array(array(1, 2), array(3, 4)); - [1,2,3,4] + > SELECT _FUNC_(array(array(1, 2), array(3, 4))); + [1, 2, 3, 4] """, since = "2.4.0") case class Flatten(child: Expression) extends UnaryExpression { @@ -2934,7 +2935,7 @@ object Sequence { examples = """ Examples: > SELECT _FUNC_('123', 2); - ['123', '123'] + ["123", "123"] """, since = "2.4.0") case class ArrayRepeat(left: Expression, right: Expression) @@ -3055,7 +3056,7 @@ case class ArrayRepeat(left: Expression, right: Expression) examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, null, 3), 3); - [1,2,null] + [1, 2, null] """, since = "2.4.0") case class ArrayRemove(left: Expression, right: Expression) extends BinaryExpression with ImplicitCastInputTypes { @@ -3245,7 +3246,7 @@ trait ArraySetLike { examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3, null, 3)); - [1,2,3,null] + [1, 2, 3, null] """, since = "2.4.0") case class ArrayDistinct(child: Expression) extends UnaryExpression with ArraySetLike with ExpectsInputTypes { @@ -3421,7 +3422,7 @@ object ArrayBinaryLike { examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); - array(1, 2, 3, 5) + [1, 2, 3, 5] """, since = "2.4.0") case class ArrayUnion(left: Expression, right: Expression) extends ArrayBinaryLike @@ -3632,7 +3633,7 @@ object ArrayUnion { examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); - array(1, 3) + [1, 3] """, since = "2.4.0") case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBinaryLike @@ -3873,7 +3874,7 @@ case class ArrayIntersect(left: Expression, right: Expression) extends ArrayBina examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array(1, 3, 5)); - array(2) + [2] """, since = "2.4.0") case class ArrayExcept(left: Expression, right: Expression) extends ArrayBinaryLike http://git-wip-us.apache.org/repos/asf/spark/blob/9c25d7f7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala index aba9c6c..117fa3e 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeCreator.scala @@ -248,8 +248,8 @@ case class CreateMap(children: Seq[Expression]) extends Expression { in keys should not be null""", examples = """ Examples: - > SELECT _FUNC_([1.0, 3.0], ['2', '4']); - {1.0:"2",3.0:"4"} + > SELECT _FUNC_(array(1.0, 3.0), array('2', '4')); + [1.0 -> "2", 3.0 -> "4"] """, since = "2.4.0") case class MapFromArrays(left: Expression, right: Expression) extends BinaryExpression with ExpectsInputTypes { http://git-wip-us.apache.org/repos/asf/spark/blob/9c25d7f7/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala index 2bb6b20..3ef2ec0 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/higherOrderFunctions.scala @@ -209,9 +209,9 @@ trait MapBasedSimpleHigherOrderFunction extends SimpleHigherOrderFunction { examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x + 1); - array(2, 3, 4) + [2, 3, 4] > SELECT _FUNC_(array(1, 2, 3), (x, i) -> x + i); - array(1, 3, 5) + [1, 3, 5] """, since = "2.4.0") case class ArrayTransform( @@ -318,7 +318,7 @@ case class MapFilter( examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), x -> x % 2 == 1); - array(1, 3) + [1, 3] """, since = "2.4.0") case class ArrayFilter( @@ -499,10 +499,10 @@ case class ArrayAggregate( usage = "_FUNC_(expr, func) - Transforms elements in a map using the function.", examples = """ Examples: - > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1); - map(array(2, 3, 4), array(1, 2, 3)) - > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v); - map(array(2, 4, 6), array(1, 2, 3)) + > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + 1); + [2 -> 1, 3 -> 2, 4 -> 3] + > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v); + [2 -> 1, 4 -> 2, 6 -> 3] """, since = "2.4.0") case class TransformKeys( @@ -549,10 +549,10 @@ case class TransformKeys( usage = "_FUNC_(expr, func) - Transforms values in the map using the function.", examples = """ Examples: - > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1); - map(array(1, 2, 3), array(2, 3, 4)) - > SELECT _FUNC_(map(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v); - map(array(1, 2, 3), array(2, 4, 6)) + > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> v + 1); + [1 -> 2, 2 -> 3, 3 -> 4] + > SELECT _FUNC_(map_from_arrays(array(1, 2, 3), array(1, 2, 3)), (k, v) -> k + v); + [1 -> 2, 2 -> 4, 3 -> 6] """, since = "2.4.0") case class TransformValues( @@ -603,7 +603,7 @@ case class TransformValues( examples = """ Examples: > SELECT _FUNC_(map(1, 'a', 2, 'b'), map(1, 'x', 2, 'y'), (k, v1, v2) -> concat(v1, v2)); - {1:"ax",2:"by"} + [1 -> "ax", 2 -> "by"] """, since = "2.4.0") case class MapZipWith(left: Expression, right: Expression, function: Expression) @@ -777,11 +777,11 @@ case class MapZipWith(left: Expression, right: Expression, function: Expression) examples = """ Examples: > SELECT _FUNC_(array(1, 2, 3), array('a', 'b', 'c'), (x, y) -> (y, x)); - array(('a', 1), ('b', 2), ('c', 3)) - > SELECT _FUNC_(array(1, 2), array(3, 4), (x, y) -> x + y)); - array(4, 6) + [["a", 1], ["b", 2], ["c", 3]] + > SELECT _FUNC_(array(1, 2), array(3, 4), (x, y) -> x + y); + [4, 6] > SELECT _FUNC_(array('a', 'b', 'c'), array('d', 'e', 'f'), (x, y) -> concat(x, y)); - array('ad', 'be', 'cf') + ["ad", "be", "cf"] """, since = "2.4.0") // scalastyle:on line.size.limit --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
