Repository: spark Updated Branches: refs/heads/master 76bd98d91 -> 0cbadf28c
[SPARK-13271][SQL] Better error message if 'path' is not specified Improved the error message as per discussion in https://github.com/apache/spark/pull/11034#discussion_r52111238. Also made `path` and `metadataPath` in FileStreamSource case insensitive. Author: Shixiong Zhu <shixi...@databricks.com> Closes #11154 from zsxwing/path. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0cbadf28 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0cbadf28 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0cbadf28 Branch: refs/heads/master Commit: 0cbadf28c99721ba1ac22ac57beef9df998ea685 Parents: 76bd98d Author: Shixiong Zhu <shixi...@databricks.com> Authored: Sun Feb 21 15:34:39 2016 -0800 Committer: Reynold Xin <r...@databricks.com> Committed: Sun Feb 21 15:34:39 2016 -0800 ---------------------------------------------------------------------- .../sql/execution/datasources/ResolvedDataSource.scala | 12 +++++++++--- .../scala/org/apache/spark/sql/sources/interfaces.scala | 5 +++-- .../spark/sql/hive/MetastoreDataSourcesSuite.scala | 2 +- 3 files changed, 13 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/0cbadf28/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ResolvedDataSource.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ResolvedDataSource.scala b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ResolvedDataSource.scala index cefa8be..eec9070 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ResolvedDataSource.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/execution/datasources/ResolvedDataSource.scala @@ -156,7 +156,9 @@ object ResolvedDataSource extends Logging { } caseInsensitiveOptions.get("paths") .map(_.split("(?<!\\\\),").map(StringUtils.unEscapeString(_, '\\', ','))) - .getOrElse(Array(caseInsensitiveOptions("path"))) + .getOrElse(Array(caseInsensitiveOptions.getOrElse("path", { + throw new IllegalArgumentException("'path' is not specified") + }))) .flatMap{ pathString => val hdfsPath = new Path(pathString) val fs = hdfsPath.getFileSystem(sqlContext.sparkContext.hadoopConfiguration) @@ -197,7 +199,9 @@ object ResolvedDataSource extends Logging { } caseInsensitiveOptions.get("paths") .map(_.split("(?<!\\\\),").map(StringUtils.unEscapeString(_, '\\', ','))) - .getOrElse(Array(caseInsensitiveOptions("path"))) + .getOrElse(Array(caseInsensitiveOptions.getOrElse("path", { + throw new IllegalArgumentException("'path' is not specified") + }))) .flatMap{ pathString => val hdfsPath = new Path(pathString) val fs = hdfsPath.getFileSystem(sqlContext.sparkContext.hadoopConfiguration) @@ -260,7 +264,9 @@ object ResolvedDataSource extends Logging { // 3. It's OK that the output path doesn't exist yet; val caseInsensitiveOptions = new CaseInsensitiveMap(options) val outputPath = { - val path = new Path(caseInsensitiveOptions("path")) + val path = new Path(caseInsensitiveOptions.getOrElse("path", { + throw new IllegalArgumentException("'path' is not specified") + })) val fs = path.getFileSystem(sqlContext.sparkContext.hadoopConfiguration) path.makeQualified(fs.getUri, fs.getWorkingDirectory) } http://git-wip-us.apache.org/repos/asf/spark/blob/0cbadf28/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala b/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala index 428a313..87ea7f5 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/sources/interfaces.scala @@ -203,10 +203,11 @@ trait HadoopFsRelationProvider extends StreamSourceProvider { schema: Option[StructType], providerName: String, parameters: Map[String, String]): Source = { - val path = parameters.getOrElse("path", { + val caseInsensitiveOptions = new CaseInsensitiveMap(parameters) + val path = caseInsensitiveOptions.getOrElse("path", { throw new IllegalArgumentException("'path' is not specified") }) - val metadataPath = parameters.getOrElse("metadataPath", s"$path/_metadata") + val metadataPath = caseInsensitiveOptions.getOrElse("metadataPath", s"$path/_metadata") def dataFrameBuilder(files: Array[String]): DataFrame = { val relation = createRelation( http://git-wip-us.apache.org/repos/asf/spark/blob/0cbadf28/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala ---------------------------------------------------------------------- diff --git a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala index 0c288bd..cd85e23 100644 --- a/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala +++ b/sql/hive/src/test/scala/org/apache/spark/sql/hive/MetastoreDataSourcesSuite.scala @@ -548,7 +548,7 @@ class MetastoreDataSourcesSuite extends QueryTest with SQLTestUtils with TestHiv "org.apache.spark.sql.json", schema, Map.empty[String, String]) - }.getMessage.contains("key not found: path"), + }.getMessage.contains("'path' is not specified"), "We should complain that path is not specified.") } } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
