Repository: spark
Updated Branches:
refs/heads/master 66636ef0b -> b4724db19
[SPARK-20425][SQL] Support a vertical display mode for Dataset.show
## What changes were proposed in this pull request?
This pr added a new display mode for `Dataset.show` to print output rows
vertically (one line per column value). In the current master, when printing
Dataset with many columns, the readability is low like;
```
scala> val df = spark.range(100).selectExpr((0 until 100).map(i => s"rand() AS
c$i"): _*)
scala> df.show(3, 0)
+------------------+------------------+------------------+-------------------+------------------+------------------+-------------------+------------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+-------------------+-------------------+-------------------+------------------+------------------+-------------------+------------------+-------------------+------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------------------+-------------------+--------------------+------------------+------------------+-------------------+-------------------+-------------------+------------------+------------------+-------------------+------------------+------------------+-------------------+-------------------+-------------------+------------------+--------------------+--------------------+-------------------+-------------------+-----------------
--+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+-------------------+------------------+------------------+-------------------+-------------------+------------------+-------------------+------------------+------------------+-----------------+-------------------+-------------------+------------------+-------------------+------------------+-------------------+-------------------+-------------------+------------------+-------------------+------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+------------------+-------------------+-------------------+------------------+------------------+------------------+-------------------+------------------+-------------------+------------------+-------------------+-------------------+-------------------+
|c0 |c1 |c2 |c3
|c4 |c5 |c6 |c7
|c8 |c9 |c10 |c11
|c12 |c13 |c14 |c15
|c16 |c17 |c18 |c19
|c20 |c21 |c22 |c23
|c24 |c25 |c26 |c27
|c28 |c29 |c30 |c31
|c32 |c33 |c34 |c35
|c36 |c37 |c38 |c39
|c40 |c41 |c42 |c43
|c44 |c45 |c46 |c47
|c48 |c49 |c50
|c51 |c52 |c53 |c54
|c55 |c56 |c57 |c58
|c59 |c60 |c61 |c62
|c63 |c64 |c65 |c66
|c67 |c68 |c69 |c70
|c71 |c72 |c73 |c74
|c75 |c76 |c77 |c78
|c79 |c80 |c81 |c82
|c83 |c84 |c85 |c86
|c87 |c88 |c89 |c90
|c91 |c92 |c93 |c94
|c95 |c96 |c97 |c98
|c99 |
+------------------+------------------+------------------+-------------------+------------------+------------------+-------------------+------------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+-------------------+-------------------+-------------------+------------------+------------------+-------------------+------------------+-------------------+------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------------------+-------------------+--------------------+------------------+------------------+-------------------+-------------------+-------------------+------------------+------------------+-------------------+------------------+------------------+-------------------+-------------------+-------------------+------------------+--------------------+--------------------+-------------------+-------------------+-----------------
--+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+-------------------+------------------+------------------+-------------------+-------------------+------------------+-------------------+------------------+------------------+-----------------+-------------------+-------------------+------------------+-------------------+------------------+-------------------+-------------------+-------------------+------------------+-------------------+------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+------------------+-------------------+-------------------+------------------+------------------+------------------+-------------------+------------------+-------------------+------------------+-------------------+-------------------+-------------------+
|0.6306087152476858|0.9174349686288383|0.5511324165035159|0.3320844128641819
|0.7738486877101489|0.2154915886962553|0.4754997600674299 |0.922780639280355
|0.7136894772661909|0.2277580838165979|0.5926874459847249|0.40311408392226633|0.467830264333843
|0.8330466896984213|0.1893258482389527|0.6320849515511165 |0.7530911056912044
|0.06700254871955424|0.370528597355559
|0.2755437445193154|0.23704391110980128|0.8067400174905822|0.13597793616251852|0.1708888820162453|0.01672725007605702|0.983118121881555
|0.25040195628629924|0.060537253723083384|0.20000530582637488|0.3400572407133511|0.9375689433322597
|0.057039316954370256|0.8053269714347623|0.5247817572228813|0.28419308820527944|0.9798908885194533
|0.31805988175678146|0.7034448027077574|0.5400575751346084|0.25336322371116216|0.9361634546853429|0.6118681368289798|0.6295081549153907
|0.13417468943957422|0.41617137072255794|0.7267230869252035|0.023792726137561115|0.5776157058356362
|0.04884204913195467|0.26728716103441275|0.646680370807925
|0.9782712690657244
|0.16434031314818154|0.20985522381321275|0.24739842475440077
|0.26335189682977334|0.19604841662422068|0.10742950487300651|0.20283136488091502|0.3100312319723688|0.886959006630645
|0.25157102269776244|0.34428775168410786|0.3500506818575777|0.3781142441912052
|0.8560316444386715|0.4737104888956839|0.735903101602148|0.02236617130529006|0.8769074095835873
|0.2001426662503153|0.5534032319238532
|0.7289496620397098|0.41955191309992157|0.9337700133660436
|0.34059094378451005|0.6419144759403556|0.08167496930341167|0.9947099478497635|0.48010888605366586|0.22314796858167918|0.17786598882331306|0.7351521162297135
|0.5422057170020095 |0.9521927872726792 |0.7459825486368227
|0.40907708791990627|0.8903819313311575|0.7251413746923618 |0.2977174938745204
|0.9515209660203555|0.9375968604766713|0.5087851740042524|0.4255237544908751
|0.8023768698664653|0.48003189618006703|0.1775841829745185|0.09050775629268382|0.6743909291138167
|0.2498415755876865 |
|0.6866473844170801|0.4774360641212433|0.631696201340726
|0.33979113021468343|0.5663049010847052|0.7280190472258865|0.41370958502324806|0.9977433873622218|0.7671957338989901|0.2788708556233931|0.3355106391656496|0.88478952319287
|0.0333974166999893|0.6061744715862606|0.9617779139652359|0.22484954822341863|0.12770906021550898|0.5577789629508672
|0.2877649024640704|0.5566577406549361|0.9334933255278052
|0.9166720585157266|0.9689249324600591 |0.6367502457478598|0.7993572745928459
|0.23213222324218108|0.11928284054154137|0.6173493362456599
|0.0505122058694798 |0.9050228629552983|0.17112767911121707|0.47395598348370005
|0.5820498657823081|0.6241124650645072|0.18587258258036776|0.14987593554122225|0.3079446253653946
|0.9414228822867968|0.8362276265462365|0.9155655305576353
|0.5121559807153562|0.8963362656525707|0.22765970274318037|0.8177039187132797
|0.8190326635933787 |0.5256005177032199|0.8167598457269669
|0.030936807130934496|0.6733006585281015 |0.4208049626816347 |0.246030857385185
38|0.22719198954208153|0.1622280557565281
|0.22217325159218038|0.014684419513742553|0.08987111517447499|0.2157764759142622
|0.8223414104088321 |0.4868624404491777
|0.4016191733088167|0.6169281906889263|0.15603611040433385|0.18289285085714913|0.9538408988218972|0.15037154865295121|0.5364516961987454|0.8077254873163031|0.712600478545675|0.7277477241003857
|0.19822912960348305|0.8305051199208777|0.18631911396566114|0.8909532487898342|0.3470409226992506
|0.35306974180587636|0.9107058868891469
|0.3321327206004986|0.48952332459050607|0.3630403307479373|0.5400046826340376
|0.5387377194310529
|0.42860539421837585|0.23214101630985995|0.21438968839794847|0.15370603160082352|0.04355605642700022|0.6096006707067466
|0.6933354157094292|0.06302172470859002|0.03174631856164001|0.664243581650643
|0.7833239547446621|0.696884598352864
|0.34626385933237736|0.9263495598791336|0.404818892816584
|0.2085585394755507|0.6150004897990109 |0.05391193524302473|0.28188484028329097|
+------------------+------------------+------------------+-------------------+------------------+------------------+-------------------+------------------+------------------+------------------+------------------+-------------------+------------------+------------------+------------------+-------------------+-------------------+-------------------+------------------+------------------+-------------------+------------------+-------------------+------------------+-------------------+-------------------+-------------------+--------------------+-------------------+------------------+-------------------+--------------------+------------------+------------------+-------------------+-------------------+-------------------+------------------+------------------+-------------------+------------------+------------------+-------------------+-------------------+-------------------+------------------+--------------------+--------------------+-------------------+-------------------+-----------------
--+-------------------+-------------------+-------------------+--------------------+-------------------+-------------------+-------------------+-------------------+------------------+------------------+-------------------+-------------------+------------------+-------------------+------------------+------------------+-----------------+-------------------+-------------------+------------------+-------------------+------------------+-------------------+-------------------+-------------------+------------------+-------------------+------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+-------------------+------------------+-------------------+-------------------+------------------+------------------+------------------+-------------------+------------------+-------------------+------------------+-------------------+-------------------+-------------------+
only showing top 2 rows
```
`psql`, CLI for PostgreSQL, supports a vertical display mode for this case like:
http://stackoverflow.com/questions/9604723/alternate-output-format-for-psql
```
-RECORD 0-------------------
c0 | 0.6306087152476858
c1 | 0.9174349686288383
c2 | 0.5511324165035159
...
c98 | 0.05391193524302473
c99 | 0.28188484028329097
-RECORD 1-------------------
c0 | 0.6866473844170801
c1 | 0.4774360641212433
c2 | 0.631696201340726
...
c98 | 0.05391193524302473
c99 | 0.28188484028329097
only showing top 2 rows
```
## How was this patch tested?
Added tests in `DataFrameSuite`.
Author: Takeshi Yamamuro <yamam...@apache.org>
Closes #17733 from maropu/SPARK-20425.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b4724db1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b4724db1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b4724db1
Branch: refs/heads/master
Commit: b4724db19a10387a803cd7beec14facf7ad1894a
Parents: 66636ef
Author: Takeshi Yamamuro <yamam...@apache.org>
Authored: Wed Apr 26 22:18:01 2017 -0700
Committer: Xiao Li <gatorsm...@gmail.com>
Committed: Wed Apr 26 22:18:01 2017 -0700
----------------------------------------------------------------------
R/pkg/R/DataFrame.R | 8 +-
python/pyspark/sql/dataframe.py | 15 +-
.../scala/org/apache/spark/sql/Dataset.scala | 149 +++++++++++++++----
.../org/apache/spark/sql/DataFrameSuite.scala | 112 ++++++++++++++
4 files changed, 247 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/b4724db1/R/pkg/R/DataFrame.R
----------------------------------------------------------------------
diff --git a/R/pkg/R/DataFrame.R b/R/pkg/R/DataFrame.R
index cd6f03a..7e57ba6 100644
--- a/R/pkg/R/DataFrame.R
+++ b/R/pkg/R/DataFrame.R
@@ -194,6 +194,7 @@ setMethod("isLocal",
#' 20 characters will be truncated. However, if set greater
than zero,
#' truncates strings longer than \code{truncate} characters
and all cells
#' will be aligned right.
+#' @param vertical whether print output rows vertically (one line per column
value).
#' @param ... further arguments to be passed to or from other methods.
#' @family SparkDataFrame functions
#' @aliases showDF,SparkDataFrame-method
@@ -210,12 +211,13 @@ setMethod("isLocal",
#' @note showDF since 1.4.0
setMethod("showDF",
signature(x = "SparkDataFrame"),
- function(x, numRows = 20, truncate = TRUE) {
+ function(x, numRows = 20, truncate = TRUE, vertical = FALSE) {
if (is.logical(truncate) && truncate) {
- s <- callJMethod(x@sdf, "showString", numToInt(numRows),
numToInt(20))
+ s <- callJMethod(x@sdf, "showString", numToInt(numRows),
numToInt(20), vertical)
} else {
truncate2 <- as.numeric(truncate)
- s <- callJMethod(x@sdf, "showString", numToInt(numRows),
numToInt(truncate2))
+ s <- callJMethod(x@sdf, "showString", numToInt(numRows),
numToInt(truncate2),
+ vertical)
}
cat(s)
})
http://git-wip-us.apache.org/repos/asf/spark/blob/b4724db1/python/pyspark/sql/dataframe.py
----------------------------------------------------------------------
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 774caf5..ff21bb5 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -290,13 +290,15 @@ class DataFrame(object):
return self._jdf.isStreaming()
@since(1.3)
- def show(self, n=20, truncate=True):
+ def show(self, n=20, truncate=True, vertical=False):
"""Prints the first ``n`` rows to the console.
:param n: Number of rows to show.
:param truncate: If set to True, truncate strings longer than 20 chars
by default.
If set to a number greater than one, truncates long strings to
length ``truncate``
and align cells right.
+ :param vertical: If set to True, print output rows vertically (one line
+ per column value).
>>> df
DataFrame[age: int, name: string]
@@ -314,11 +316,18 @@ class DataFrame(object):
| 2| Ali|
| 5| Bob|
+---+----+
+ >>> df.show(vertical=True)
+ -RECORD 0-----
+ age | 2
+ name | Alice
+ -RECORD 1-----
+ age | 5
+ name | Bob
"""
if isinstance(truncate, bool) and truncate:
- print(self._jdf.showString(n, 20))
+ print(self._jdf.showString(n, 20, vertical))
else:
- print(self._jdf.showString(n, int(truncate)))
+ print(self._jdf.showString(n, int(truncate), vertical))
def __repr__(self):
return "DataFrame[%s]" % (", ".join("%s: %s" % c for c in self.dtypes))
http://git-wip-us.apache.org/repos/asf/spark/blob/b4724db1/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
index 06dd550..147e765 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/Dataset.scala
@@ -240,8 +240,10 @@ class Dataset[T] private[sql](
* @param _numRows Number of rows to show
* @param truncate If set to more than 0, truncates strings to `truncate`
characters and
* all cells will be aligned right.
+ * @param vertical If set to true, prints output rows vertically (one line
per column value).
*/
- private[sql] def showString(_numRows: Int, truncate: Int = 20): String = {
+ private[sql] def showString(
+ _numRows: Int, truncate: Int = 20, vertical: Boolean = false): String = {
val numRows = _numRows.max(0)
val takeResult = toDF().take(numRows + 1)
val hasMoreData = takeResult.length > numRows
@@ -277,46 +279,80 @@ class Dataset[T] private[sql](
val sb = new StringBuilder
val numCols = schema.fieldNames.length
+ // We set a minimum column width at '3'
+ val minimumColWidth = 3
- // Initialise the width of each column to a minimum value of '3'
- val colWidths = Array.fill(numCols)(3)
+ if (!vertical) {
+ // Initialise the width of each column to a minimum value
+ val colWidths = Array.fill(numCols)(minimumColWidth)
- // Compute the width of each column
- for (row <- rows) {
- for ((cell, i) <- row.zipWithIndex) {
- colWidths(i) = math.max(colWidths(i), cell.length)
- }
- }
-
- // Create SeparateLine
- val sep: String = colWidths.map("-" * _).addString(sb, "+", "+",
"+\n").toString()
-
- // column names
- rows.head.zipWithIndex.map { case (cell, i) =>
- if (truncate > 0) {
- StringUtils.leftPad(cell, colWidths(i))
- } else {
- StringUtils.rightPad(cell, colWidths(i))
+ // Compute the width of each column
+ for (row <- rows) {
+ for ((cell, i) <- row.zipWithIndex) {
+ colWidths(i) = math.max(colWidths(i), cell.length)
+ }
}
- }.addString(sb, "|", "|", "|\n")
- sb.append(sep)
+ // Create SeparateLine
+ val sep: String = colWidths.map("-" * _).addString(sb, "+", "+",
"+\n").toString()
- // data
- rows.tail.map {
- _.zipWithIndex.map { case (cell, i) =>
+ // column names
+ rows.head.zipWithIndex.map { case (cell, i) =>
if (truncate > 0) {
- StringUtils.leftPad(cell.toString, colWidths(i))
+ StringUtils.leftPad(cell, colWidths(i))
} else {
- StringUtils.rightPad(cell.toString, colWidths(i))
+ StringUtils.rightPad(cell, colWidths(i))
}
}.addString(sb, "|", "|", "|\n")
- }
- sb.append(sep)
+ sb.append(sep)
+
+ // data
+ rows.tail.foreach {
+ _.zipWithIndex.map { case (cell, i) =>
+ if (truncate > 0) {
+ StringUtils.leftPad(cell.toString, colWidths(i))
+ } else {
+ StringUtils.rightPad(cell.toString, colWidths(i))
+ }
+ }.addString(sb, "|", "|", "|\n")
+ }
+
+ sb.append(sep)
+ } else {
+ // Extended display mode enabled
+ val fieldNames = rows.head
+ val dataRows = rows.tail
+
+ // Compute the width of field name and data columns
+ val fieldNameColWidth = fieldNames.foldLeft(minimumColWidth) { case
(curMax, fieldName) =>
+ math.max(curMax, fieldName.length)
+ }
+ val dataColWidth = dataRows.foldLeft(minimumColWidth) { case (curMax,
row) =>
+ math.max(curMax, row.map(_.length).reduceLeftOption[Int] { case
(cellMax, cell) =>
+ math.max(cellMax, cell)
+ }.getOrElse(0))
+ }
+
+ dataRows.zipWithIndex.foreach { case (row, i) =>
+ // "+ 5" in size means a character length except for padded names and
data
+ val rowHeader = StringUtils.rightPad(
+ s"-RECORD $i", fieldNameColWidth + dataColWidth + 5, "-")
+ sb.append(rowHeader).append("\n")
+ row.zipWithIndex.map { case (cell, j) =>
+ val fieldName = StringUtils.rightPad(fieldNames(j),
fieldNameColWidth)
+ val data = StringUtils.rightPad(cell, dataColWidth)
+ s" $fieldName | $data "
+ }.addString(sb, "", "\n", "\n")
+ }
+ }
- // For Data that has more than "numRows" records
- if (hasMoreData) {
+ // Print a footer
+ if (vertical && data.isEmpty) {
+ // In a vertical mode, print an empty row set explicitly
+ sb.append("(0 rows)\n")
+ } else if (hasMoreData) {
+ // For Data that has more than "numRows" records
val rowsString = if (numRows == 1) "row" else "rows"
sb.append(s"only showing top $numRows $rowsString\n")
}
@@ -663,8 +699,59 @@ class Dataset[T] private[sql](
* @group action
* @since 1.6.0
*/
+ def show(numRows: Int, truncate: Int): Unit = show(numRows, truncate,
vertical = false)
+
+ /**
+ * Displays the Dataset in a tabular form. For example:
+ * {{{
+ * year month AVG('Adj Close) MAX('Adj Close)
+ * 1980 12 0.503218 0.595103
+ * 1981 01 0.523289 0.570307
+ * 1982 02 0.436504 0.475256
+ * 1983 03 0.410516 0.442194
+ * 1984 04 0.450090 0.483521
+ * }}}
+ *
+ * If `vertical` enabled, this command prints output rows vertically (one
line per column value)?
+ *
+ * {{{
+ * -RECORD 0-------------------
+ * year | 1980
+ * month | 12
+ * AVG('Adj Close) | 0.503218
+ * AVG('Adj Close) | 0.595103
+ * -RECORD 1-------------------
+ * year | 1981
+ * month | 01
+ * AVG('Adj Close) | 0.523289
+ * AVG('Adj Close) | 0.570307
+ * -RECORD 2-------------------
+ * year | 1982
+ * month | 02
+ * AVG('Adj Close) | 0.436504
+ * AVG('Adj Close) | 0.475256
+ * -RECORD 3-------------------
+ * year | 1983
+ * month | 03
+ * AVG('Adj Close) | 0.410516
+ * AVG('Adj Close) | 0.442194
+ * -RECORD 4-------------------
+ * year | 1984
+ * month | 04
+ * AVG('Adj Close) | 0.450090
+ * AVG('Adj Close) | 0.483521
+ * }}}
+ *
+ * @param numRows Number of rows to show
+ * @param truncate If set to more than 0, truncates strings to `truncate`
characters and
+ * all cells will be aligned right.
+ * @param vertical If set to true, prints output rows vertically (one line
per column value).
+ * @group action
+ * @since 2.3.0
+ */
// scalastyle:off println
- def show(numRows: Int, truncate: Int): Unit = println(showString(numRows,
truncate))
+ def show(numRows: Int, truncate: Int, vertical: Boolean): Unit =
+ println(showString(numRows, truncate, vertical))
// scalastyle:on println
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/b4724db1/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index b4893b5..ef0de6f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -764,6 +764,21 @@ class DataFrameSuite extends QueryTest with
SharedSQLContext {
assert(df.showString(10, truncate = 20) === expectedAnswerForTrue)
}
+ test("showString: truncate = [0, 20], vertical = true") {
+ val longString = Array.fill(21)("1").mkString
+ val df = sparkContext.parallelize(Seq("1", longString)).toDF()
+ val expectedAnswerForFalse = "-RECORD 0----------------------\n" +
+ " value | 1 \n" +
+ "-RECORD 1----------------------\n" +
+ " value | 111111111111111111111 \n"
+ assert(df.showString(10, truncate = 0, vertical = true) ===
expectedAnswerForFalse)
+ val expectedAnswerForTrue = "-RECORD 0---------------------\n" +
+ " value | 1 \n" +
+ "-RECORD 1---------------------\n" +
+ " value | 11111111111111111... \n"
+ assert(df.showString(10, truncate = 20, vertical = true) ===
expectedAnswerForTrue)
+ }
+
test("showString: truncate = [3, 17]") {
val longString = Array.fill(21)("1").mkString
val df = sparkContext.parallelize(Seq("1", longString)).toDF()
@@ -785,6 +800,21 @@ class DataFrameSuite extends QueryTest with
SharedSQLContext {
assert(df.showString(10, truncate = 17) === expectedAnswerForTrue)
}
+ test("showString: truncate = [3, 17], vertical = true") {
+ val longString = Array.fill(21)("1").mkString
+ val df = sparkContext.parallelize(Seq("1", longString)).toDF()
+ val expectedAnswerForFalse = "-RECORD 0----\n" +
+ " value | 1 \n" +
+ "-RECORD 1----\n" +
+ " value | 111 \n"
+ assert(df.showString(10, truncate = 3, vertical = true) ===
expectedAnswerForFalse)
+ val expectedAnswerForTrue = "-RECORD 0------------------\n" +
+ " value | 1 \n" +
+ "-RECORD 1------------------\n" +
+ " value | 11111111111111... \n"
+ assert(df.showString(10, truncate = 17, vertical = true) ===
expectedAnswerForTrue)
+ }
+
test("showString(negative)") {
val expectedAnswer = """+---+-----+
||key|value|
@@ -795,6 +825,11 @@ class DataFrameSuite extends QueryTest with
SharedSQLContext {
assert(testData.select($"*").showString(-1) === expectedAnswer)
}
+ test("showString(negative), vertical = true") {
+ val expectedAnswer = "(0 rows)\n"
+ assert(testData.select($"*").showString(-1, vertical = true) ===
expectedAnswer)
+ }
+
test("showString(0)") {
val expectedAnswer = """+---+-----+
||key|value|
@@ -805,6 +840,11 @@ class DataFrameSuite extends QueryTest with
SharedSQLContext {
assert(testData.select($"*").showString(0) === expectedAnswer)
}
+ test("showString(0), vertical = true") {
+ val expectedAnswer = "(0 rows)\n"
+ assert(testData.select($"*").showString(0, vertical = true) ===
expectedAnswer)
+ }
+
test("showString: array") {
val df = Seq(
(Array(1, 2, 3), Array(1, 2, 3)),
@@ -820,6 +860,20 @@ class DataFrameSuite extends QueryTest with
SharedSQLContext {
assert(df.showString(10) === expectedAnswer)
}
+ test("showString: array, vertical = true") {
+ val df = Seq(
+ (Array(1, 2, 3), Array(1, 2, 3)),
+ (Array(2, 3, 4), Array(2, 3, 4))
+ ).toDF()
+ val expectedAnswer = "-RECORD 0--------\n" +
+ " _1 | [1, 2, 3] \n" +
+ " _2 | [1, 2, 3] \n" +
+ "-RECORD 1--------\n" +
+ " _1 | [2, 3, 4] \n" +
+ " _2 | [2, 3, 4] \n"
+ assert(df.showString(10, vertical = true) === expectedAnswer)
+ }
+
test("showString: binary") {
val df = Seq(
("12".getBytes(StandardCharsets.UTF_8),
"ABC.".getBytes(StandardCharsets.UTF_8)),
@@ -835,6 +889,20 @@ class DataFrameSuite extends QueryTest with
SharedSQLContext {
assert(df.showString(10) === expectedAnswer)
}
+ test("showString: binary, vertical = true") {
+ val df = Seq(
+ ("12".getBytes(StandardCharsets.UTF_8),
"ABC.".getBytes(StandardCharsets.UTF_8)),
+ ("34".getBytes(StandardCharsets.UTF_8),
"12346".getBytes(StandardCharsets.UTF_8))
+ ).toDF()
+ val expectedAnswer = "-RECORD 0---------------\n" +
+ " _1 | [31 32] \n" +
+ " _2 | [41 42 43 2E] \n" +
+ "-RECORD 1---------------\n" +
+ " _1 | [33 34] \n" +
+ " _2 | [31 32 33 34 36] \n"
+ assert(df.showString(10, vertical = true) === expectedAnswer)
+ }
+
test("showString: minimum column width") {
val df = Seq(
(1, 1),
@@ -850,6 +918,20 @@ class DataFrameSuite extends QueryTest with
SharedSQLContext {
assert(df.showString(10) === expectedAnswer)
}
+ test("showString: minimum column width, vertical = true") {
+ val df = Seq(
+ (1, 1),
+ (2, 2)
+ ).toDF()
+ val expectedAnswer = "-RECORD 0--\n" +
+ " _1 | 1 \n" +
+ " _2 | 1 \n" +
+ "-RECORD 1--\n" +
+ " _1 | 2 \n" +
+ " _2 | 2 \n"
+ assert(df.showString(10, vertical = true) === expectedAnswer)
+ }
+
test("SPARK-7319 showString") {
val expectedAnswer = """+---+-----+
||key|value|
@@ -861,6 +943,14 @@ class DataFrameSuite extends QueryTest with
SharedSQLContext {
assert(testData.select($"*").showString(1) === expectedAnswer)
}
+ test("SPARK-7319 showString, vertical = true") {
+ val expectedAnswer = "-RECORD 0----\n" +
+ " key | 1 \n" +
+ " value | 1 \n" +
+ "only showing top 1 row\n"
+ assert(testData.select($"*").showString(1, vertical = true) ===
expectedAnswer)
+ }
+
test("SPARK-7327 show with empty dataFrame") {
val expectedAnswer = """+---+-----+
||key|value|
@@ -870,6 +960,10 @@ class DataFrameSuite extends QueryTest with
SharedSQLContext {
assert(testData.select($"*").filter($"key" < 0).showString(1) ===
expectedAnswer)
}
+ test("SPARK-7327 show with empty dataFrame, vertical = true") {
+ assert(testData.select($"*").filter($"key" < 0).showString(1, vertical =
true) === "(0 rows)\n")
+ }
+
test("SPARK-18350 show with session local timezone") {
val d = Date.valueOf("2016-12-01")
val ts = Timestamp.valueOf("2016-12-01 00:00:00")
@@ -894,6 +988,24 @@ class DataFrameSuite extends QueryTest with
SharedSQLContext {
}
}
+ test("SPARK-18350 show with session local timezone, vertical = true") {
+ val d = Date.valueOf("2016-12-01")
+ val ts = Timestamp.valueOf("2016-12-01 00:00:00")
+ val df = Seq((d, ts)).toDF("d", "ts")
+ val expectedAnswer = "-RECORD 0------------------\n" +
+ " d | 2016-12-01 \n" +
+ " ts | 2016-12-01 00:00:00 \n"
+ assert(df.showString(1, truncate = 0, vertical = true) === expectedAnswer)
+
+ withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "GMT") {
+
+ val expectedAnswer = "-RECORD 0------------------\n" +
+ " d | 2016-12-01 \n" +
+ " ts | 2016-12-01 08:00:00 \n"
+ assert(df.showString(1, truncate = 0, vertical = true) ===
expectedAnswer)
+ }
+ }
+
test("createDataFrame(RDD[Row], StructType) should convert UDTs
(SPARK-6672)") {
val rowRDD = sparkContext.parallelize(Seq(Row(new ExamplePoint(1.0, 2.0))))
val schema = StructType(Array(StructField("point", new ExamplePointUDT(),
false)))
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
