Repository: spark Updated Branches: refs/heads/master e52691398 -> eb10b481c
[SPARK-15286][SQL] Make the output readable for EXPLAIN CREATE TABLE and DESC EXTENDED #### What changes were proposed in this pull request? Before this PR, the output of EXPLAIN of following SQL is like ```SQL CREATE EXTERNAL TABLE extTable_with_partitions (key INT, value STRING) PARTITIONED BY (ds STRING, hr STRING) LOCATION '/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-b39a6185-8981-403b-a4aa-36fb2f4ca8a9' ``` ``ExecutedCommand CreateTableCommand CatalogTable(`extTable_with_partitions`,CatalogTableType(EXTERNAL),CatalogStorageFormat(Some(/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-dd234718-e85d-4c5a-8353-8f1834ac0323),Some(org.apache.hadoop.mapred.TextInputFormat),Some(org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat),None,false,Map()),List(CatalogColumn(key,int,true,None), CatalogColumn(value,string,true,None), CatalogColumn(ds,string,true,None), CatalogColumn(hr,string,true,None)),List(ds, hr),List(),List(),-1,,1463026413544,-1,Map(),None,None,None), false`` After this PR, the output is like ``` ExecutedCommand : +- CreateTableCommand CatalogTable( Table:`extTable_with_partitions` Created:Thu Jun 02 21:30:54 PDT 2016 Last Access:Wed Dec 31 15:59:59 PST 1969 Type:EXTERNAL Schema:[`key` int, `value` string, `ds` string, `hr` string] Partition Columns:[`ds`, `hr`] Storage(Location:/private/var/folders/4b/sgmfldk15js406vk7lw5llzw0000gn/T/spark-a06083b8-8e88-4d07-9ff0-d6bd8d943ad3, InputFormat:org.apache.hadoop.mapred.TextInputFormat, OutputFormat:org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat)), false ``` This is also applicable to `DESC EXTENDED`. However, this does not have special handling for Data Source Tables. If needed, we need to move the logics of `DDLUtil`. Let me know if we should do it in this PR. Thanks! rxin liancheng #### How was this patch tested? Manual testing Author: gatorsmile <gatorsm...@gmail.com> Closes #13070 from gatorsmile/betterExplainCatalogTable. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/eb10b481 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/eb10b481 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/eb10b481 Branch: refs/heads/master Commit: eb10b481ca51a3e010c168a1fb0baf1a94dcadfa Parents: e526913 Author: gatorsmile <gatorsm...@gmail.com> Authored: Fri Jun 3 13:56:22 2016 -0700 Committer: Wenchen Fan <wenc...@databricks.com> Committed: Fri Jun 3 13:56:22 2016 -0700 ---------------------------------------------------------------------- .../spark/sql/catalyst/catalog/interface.scala | 60 +++++++++++++++++++- 1 file changed, 58 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/eb10b481/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala index b591957..6197aca 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/catalog/interface.scala @@ -17,6 +17,7 @@ package org.apache.spark.sql.catalyst.catalog +import java.util.Date import javax.annotation.Nullable import org.apache.spark.sql.AnalysisException @@ -48,7 +49,26 @@ case class CatalogStorageFormat( outputFormat: Option[String], serde: Option[String], compressed: Boolean, - serdeProperties: Map[String, String]) + serdeProperties: Map[String, String]) { + + override def toString: String = { + val serdePropsToString = + if (serdeProperties.nonEmpty) { + s"Properties: " + serdeProperties.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]") + } else { + "" + } + val output = + Seq(locationUri.map("Location: " + _).getOrElse(""), + inputFormat.map("InputFormat: " + _).getOrElse(""), + outputFormat.map("OutputFormat: " + _).getOrElse(""), + if (compressed) "Compressed" else "", + serde.map("Serde: " + _).getOrElse(""), + serdePropsToString) + output.filter(_.nonEmpty).mkString("Storage(", ", ", ")") + } + +} object CatalogStorageFormat { /** Empty storage format for default values and copies. */ @@ -65,8 +85,18 @@ case class CatalogColumn( // as a string due to issues in converting Hive varchars to and from SparkSQL strings. @Nullable dataType: String, nullable: Boolean = true, - comment: Option[String] = None) + comment: Option[String] = None) { + + override def toString: String = { + val output = + Seq(s"`$name`", + dataType, + if (!nullable) "NOT NULL" else "", + comment.map("(" + _ + ")").getOrElse("")) + output.filter(_.nonEmpty).mkString(" ") + } +} /** * A partition (Hive style) defined in the catalog. @@ -140,6 +170,32 @@ case class CatalogTable( locationUri, inputFormat, outputFormat, serde, compressed, serdeProperties)) } + override def toString: String = { + val tableProperties = properties.map(p => p._1 + "=" + p._2).mkString("[", ", ", "]") + val partitionColumns = partitionColumnNames.map("`" + _ + "`").mkString("[", ", ", "]") + val sortColumns = sortColumnNames.map("`" + _ + "`").mkString("[", ", ", "]") + val bucketColumns = bucketColumnNames.map("`" + _ + "`").mkString("[", ", ", "]") + + val output = + Seq(s"Table: ${identifier.quotedString}", + if (owner.nonEmpty) s"Owner: $owner" else "", + s"Created: ${new Date(createTime).toString}", + s"Last Access: ${new Date(lastAccessTime).toString}", + s"Type: ${tableType.name}", + if (schema.nonEmpty) s"Schema: ${schema.mkString("[", ", ", "]")}" else "", + if (partitionColumnNames.nonEmpty) s"Partition Columns: $partitionColumns" else "", + if (numBuckets != -1) s"Num Buckets: $numBuckets" else "", + if (bucketColumnNames.nonEmpty) s"Bucket Columns: $bucketColumns" else "", + if (sortColumnNames.nonEmpty) s"Sort Columns: $sortColumns" else "", + viewOriginalText.map("Original View: " + _).getOrElse(""), + viewText.map("View: " + _).getOrElse(""), + comment.map("Comment: " + _).getOrElse(""), + if (properties.nonEmpty) s"Properties: $tableProperties" else "", + s"$storage") + + output.filter(_.nonEmpty).mkString("CatalogTable(\n\t", "\n\t", ")") + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org