spark git commit: [SPARK-16415][SQL] fix catalog string error

2016-07-07 Thread rxin 
Repository: spark
Updated Branches:
  refs/heads/branch-2.0 cbfd94eac -> 30cb3f1d3


[SPARK-16415][SQL] fix catalog string error

## What changes were proposed in this pull request?

In #13537 we truncate `simpleString` if it is a long `StructType`. But 
sometimes we need `catalogString` to reconstruct `TypeInfo`, for example in 
description of [SPARK-16415 
](https://issues.apache.org/jira/browse/SPARK-16415). So we need to keep the 
implementation of `catalogString` not affected by our truncate.

## How was this patch tested?

added a test case.

Author: Daoyuan Wang 

Closes #14089 from adrian-wang/catalogstring.

(cherry picked from commit 28710b42b0d18a55bd64d597558649537259b127)
Signed-off-by: Reynold Xin 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/30cb3f1d
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/30cb3f1d
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/30cb3f1d

Branch: refs/heads/branch-2.0
Commit: 30cb3f1d3a1d413568d586e6b8df56f74f05d80e
Parents: cbfd94e
Author: Daoyuan Wang 
Authored: Thu Jul 7 11:08:06 2016 -0700
Committer: Reynold Xin 
Committed: Thu Jul 7 11:08:12 2016 -0700

--
 .../scala/org/apache/spark/sql/types/StructType.scala |  6 ++
 .../spark/sql/hive/HiveMetastoreCatalogSuite.scala| 14 +++---
 2 files changed, 17 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/30cb3f1d/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
index effef54..55fdfbe 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
@@ -298,6 +298,12 @@ case class StructType(fields: Array[StructField]) extends 
DataType with Seq[Stru
 Utils.truncatedString(fieldTypes, "struct<", ",", ">")
   }
 
+  override def catalogString: String = {
+// in catalogString, we should not truncate
+val fieldTypes = fields.map(field => 
s"${field.name}:${field.dataType.catalogString}")
+s"struct<${fieldTypes.mkString(",")}>"
+  }
+
   override def sql: String = {
 val fieldTypes = fields.map(f => s"${quoteIdentifier(f.name)}: 
${f.dataType.sql}")
 s"STRUCT<${fieldTypes.mkString(", ")}>"

http://git-wip-us.apache.org/repos/asf/spark/blob/30cb3f1d/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
--
diff --git 
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
 
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index b420781..754aabb 100644
--- 
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++ 
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -26,15 +26,15 @@ import 
org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.{ExamplePointUDT, SQLTestUtils}
-import org.apache.spark.sql.types.{DecimalType, StringType, StructType}
+import org.apache.spark.sql.types.{DecimalType, StringType, StructField, 
StructType}
 
 class HiveMetastoreCatalogSuite extends TestHiveSingleton {
   import spark.implicits._
 
   test("struct field should accept underscore in sub-column name") {
 val hiveTypeStr = "struct"
-val dateType = CatalystSqlParser.parseDataType(hiveTypeStr)
-assert(dateType.isInstanceOf[StructType])
+val dataType = CatalystSqlParser.parseDataType(hiveTypeStr)
+assert(dataType.isInstanceOf[StructType])
   }
 
   test("udt to metastore type conversion") {
@@ -49,6 +49,14 @@ class HiveMetastoreCatalogSuite extends TestHiveSingleton {
 logInfo(df.queryExecution.toString)
 df.as('a).join(df.as('b), $"a.key" === $"b.key")
   }
+
+  test("should not truncate struct type catalog string") {
+def field(n: Int): StructField = {
+  StructField("col" + n, StringType)
+}
+val dataType = StructType((1 to 100).map(field))
+assert(CatalystSqlParser.parseDataType(dataType.catalogString) == dataType)
+  }
 }
 
 class DataSourceWithHiveMetastoreCatalogSuite


-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

spark git commit: [SPARK-16415][SQL] fix catalog string error

2016-07-07 Thread rxin 
Repository: spark
Updated Branches:
  refs/heads/master 0f7175def -> 28710b42b


[SPARK-16415][SQL] fix catalog string error

## What changes were proposed in this pull request?

In #13537 we truncate `simpleString` if it is a long `StructType`. But 
sometimes we need `catalogString` to reconstruct `TypeInfo`, for example in 
description of [SPARK-16415 
](https://issues.apache.org/jira/browse/SPARK-16415). So we need to keep the 
implementation of `catalogString` not affected by our truncate.

## How was this patch tested?

added a test case.

Author: Daoyuan Wang 

Closes #14089 from adrian-wang/catalogstring.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/28710b42
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/28710b42
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/28710b42

Branch: refs/heads/master
Commit: 28710b42b0d18a55bd64d597558649537259b127
Parents: 0f7175d
Author: Daoyuan Wang 
Authored: Thu Jul 7 11:08:06 2016 -0700
Committer: Reynold Xin 
Committed: Thu Jul 7 11:08:06 2016 -0700

--
 .../scala/org/apache/spark/sql/types/StructType.scala |  6 ++
 .../spark/sql/hive/HiveMetastoreCatalogSuite.scala| 14 +++---
 2 files changed, 17 insertions(+), 3 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/28710b42/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
index 0c2ebb0..dd4c88c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/StructType.scala
@@ -333,6 +333,12 @@ case class StructType(fields: Array[StructField]) extends 
DataType with Seq[Stru
 Utils.truncatedString(fieldTypes, "struct<", ",", ">")
   }
 
+  override def catalogString: String = {
+// in catalogString, we should not truncate
+val fieldTypes = fields.map(field => 
s"${field.name}:${field.dataType.catalogString}")
+s"struct<${fieldTypes.mkString(",")}>"
+  }
+
   override def sql: String = {
 val fieldTypes = fields.map(f => s"${quoteIdentifier(f.name)}: 
${f.dataType.sql}")
 s"STRUCT<${fieldTypes.mkString(", ")}>"

http://git-wip-us.apache.org/repos/asf/spark/blob/28710b42/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
--
diff --git 
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
 
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
index b420781..754aabb 100644
--- 
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
+++ 
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/HiveMetastoreCatalogSuite.scala
@@ -26,15 +26,15 @@ import 
org.apache.spark.sql.catalyst.parser.CatalystSqlParser
 import org.apache.spark.sql.hive.test.TestHiveSingleton
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.{ExamplePointUDT, SQLTestUtils}
-import org.apache.spark.sql.types.{DecimalType, StringType, StructType}
+import org.apache.spark.sql.types.{DecimalType, StringType, StructField, 
StructType}
 
 class HiveMetastoreCatalogSuite extends TestHiveSingleton {
   import spark.implicits._
 
   test("struct field should accept underscore in sub-column name") {
 val hiveTypeStr = "struct"
-val dateType = CatalystSqlParser.parseDataType(hiveTypeStr)
-assert(dateType.isInstanceOf[StructType])
+val dataType = CatalystSqlParser.parseDataType(hiveTypeStr)
+assert(dataType.isInstanceOf[StructType])
   }
 
   test("udt to metastore type conversion") {
@@ -49,6 +49,14 @@ class HiveMetastoreCatalogSuite extends TestHiveSingleton {
 logInfo(df.queryExecution.toString)
 df.as('a).join(df.as('b), $"a.key" === $"b.key")
   }
+
+  test("should not truncate struct type catalog string") {
+def field(n: Int): StructField = {
+  StructField("col" + n, StringType)
+}
+val dataType = StructType((1 to 100).map(field))
+assert(CatalystSqlParser.parseDataType(dataType.catalogString) == dataType)
+  }
 }
 
 class DataSourceWithHiveMetastoreCatalogSuite


-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org