Repository: spark Updated Branches: refs/heads/master ec5c4279e -> 60698801e
[SPARK-4618][SQL] Make foreign DDL commands options case-insensitive Using lowercase for ```options``` key to make it case-insensitive, then we should use lower case to get value from parameters. So flowing cmd work ``` create temporary table normal_parquet USING org.apache.spark.sql.parquet OPTIONS ( PATH '/xxx/data' ) ``` Author: scwf <wangf...@huawei.com> Author: wangfei <wangf...@huawei.com> Closes #3470 from scwf/ddl-ulcase and squashes the following commits: ae78509 [scwf] address comments 8f4f585 [wangfei] address comments 3c132ef [scwf] minor fix a0fc20b [scwf] Merge branch 'master' of https://github.com/apache/spark into ddl-ulcase 4f86401 [scwf] adding CaseInsensitiveMap e244e8d [wangfei] using lower case in json e0cb017 [wangfei] make options in-casesensitive Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/60698801 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/60698801 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/60698801 Branch: refs/heads/master Commit: 60698801ebc4c4947cfc7f46762a7cca2ed40452 Parents: ec5c427 Author: scwf <wangf...@huawei.com> Authored: Tue Dec 16 21:26:36 2014 -0800 Committer: Michael Armbrust <mich...@databricks.com> Committed: Tue Dec 16 21:26:36 2014 -0800 ---------------------------------------------------------------------- .../scala/org/apache/spark/sql/sources/ddl.scala | 19 ++++++++++++++++++- .../apache/spark/sql/sources/interfaces.scala | 6 +++++- .../spark/sql/sources/TableScanSuite.scala | 6 +++--- 3 files changed, 26 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/60698801/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala index ca510cb..8a66ac3 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/ddl.scala @@ -100,9 +100,26 @@ private[sql] case class CreateTableUsing( } } val dataSource = clazz.newInstance().asInstanceOf[org.apache.spark.sql.sources.RelationProvider] - val relation = dataSource.createRelation(sqlContext, options) + val relation = dataSource.createRelation(sqlContext, new CaseInsensitiveMap(options)) sqlContext.baseRelationToSchemaRDD(relation).registerTempTable(tableName) Seq.empty } } + +/** + * Builds a map in which keys are case insensitive + */ +protected class CaseInsensitiveMap(map: Map[String, String]) extends Map[String, String] { + + val baseMap = map.map(kv => kv.copy(_1 = kv._1.toLowerCase)) + + override def get(k: String): Option[String] = baseMap.get(k.toLowerCase) + + override def + [B1 >: String](kv: (String, B1)): Map[String, B1] = + baseMap + kv.copy(_1 = kv._1.toLowerCase) + + override def iterator: Iterator[(String, String)] = baseMap.iterator + + override def -(key: String): Map[String, String] = baseMap - key.toLowerCase() +} http://git-wip-us.apache.org/repos/asf/spark/blob/60698801/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala index 939b4e1..02eff80 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala @@ -36,7 +36,11 @@ import org.apache.spark.sql.catalyst.expressions.{Expression, Attribute} */ @DeveloperApi trait RelationProvider { - /** Returns a new base relation with the given parameters. */ + /** + * Returns a new base relation with the given parameters. + * Note: the parameters' keywords are case insensitive and this insensitivity is enforced + * by the Map that is passed to the function. + */ def createRelation(sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation } http://git-wip-us.apache.org/repos/asf/spark/blob/60698801/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala index b254b06..3cd7b01 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/sources/TableScanSuite.scala @@ -25,7 +25,7 @@ class SimpleScanSource extends RelationProvider { override def createRelation( sqlContext: SQLContext, parameters: Map[String, String]): BaseRelation = { - SimpleScan(parameters("from").toInt, parameters("to").toInt)(sqlContext) + SimpleScan(parameters("from").toInt, parameters("TO").toInt)(sqlContext) } } @@ -47,8 +47,8 @@ class TableScanSuite extends DataSourceTest { |CREATE TEMPORARY TABLE oneToTen |USING org.apache.spark.sql.sources.SimpleScanSource |OPTIONS ( - | from '1', - | to '10' + | From '1', + | To '10' |) """.stripMargin) } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org