Repository: spark Updated Branches: refs/heads/branch-2.0 b430aa98c -> 2cb84dd23
[SPARK-15565][SQL] Add the File Scheme to the Default Value of WAREHOUSE_PATH #### What changes were proposed in this pull request? The default value of `spark.sql.warehouse.dir` is `System.getProperty("user.dir")/spark-warehouse`. Since `System.getProperty("user.dir")` is a local dir, we should explicitly set the scheme to local filesystem. cc yhuai #### How was this patch tested? Added two test cases Author: gatorsmile <gatorsm...@gmail.com> Closes #13348 from gatorsmile/addSchemeToDefaultWarehousePath. (cherry picked from commit c17272902c95290beca274ee6316a8a98fd7a725) Signed-off-by: Yin Huai <yh...@databricks.com> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2cb84dd2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2cb84dd2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2cb84dd2 Branch: refs/heads/branch-2.0 Commit: 2cb84dd2356e782b9e606cd126057726fcf6f228 Parents: b430aa9 Author: gatorsmile <gatorsm...@gmail.com> Authored: Fri May 27 09:54:31 2016 -0700 Committer: Yin Huai <yh...@databricks.com> Committed: Fri May 27 09:54:43 2016 -0700 ---------------------------------------------------------------------- .../org/apache/spark/sql/internal/SQLConf.scala | 2 +- .../spark/sql/execution/command/DDLSuite.scala | 25 ++++++++++++++++++++ .../spark/sql/internal/SQLConfSuite.scala | 12 ++++++++++ 3 files changed, 38 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/2cb84dd2/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala index 4efefda..d1db0dd 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala @@ -55,7 +55,7 @@ object SQLConf { val WAREHOUSE_PATH = SQLConfigBuilder("spark.sql.warehouse.dir") .doc("The default location for managed databases and tables.") .stringConf - .createWithDefault("${system:user.dir}/spark-warehouse") + .createWithDefault("file:${system:user.dir}/spark-warehouse") val OPTIMIZER_MAX_ITERATIONS = SQLConfigBuilder("spark.sql.optimizer.maxIterations") .internal() http://git-wip-us.apache.org/repos/asf/spark/blob/2cb84dd2/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala index e32521a..e975756 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/command/DDLSuite.scala @@ -171,6 +171,31 @@ class DDLSuite extends QueryTest with SharedSQLContext with BeforeAndAfterEach { } } + test("Create Database using Default Warehouse Path") { + withSQLConf(SQLConf.WAREHOUSE_PATH.key -> "") { + // Will use the default location if and only if we unset the conf + spark.conf.unset(SQLConf.WAREHOUSE_PATH.key) + val catalog = spark.sessionState.catalog + val dbName = "db1" + try { + sql(s"CREATE DATABASE $dbName") + val db1 = catalog.getDatabaseMetadata(dbName) + val expectedLocation = + "file:" + appendTrailingSlash(System.getProperty("user.dir")) + + s"spark-warehouse/$dbName.db" + assert(db1 == CatalogDatabase( + dbName, + "", + expectedLocation, + Map.empty)) + sql(s"DROP DATABASE $dbName CASCADE") + assert(!catalog.databaseExists(dbName)) + } finally { + catalog.reset() + } + } + } + test("Create/Drop Database - location") { val catalog = spark.sessionState.catalog val databaseNames = Seq("db1", "`database`") http://git-wip-us.apache.org/repos/asf/spark/blob/2cb84dd2/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala index ad5365a..3d4fc75 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/internal/SQLConfSuite.scala @@ -207,4 +207,16 @@ class SQLConfSuite extends QueryTest with SharedSQLContext { } } + test("default value of WAREHOUSE_PATH") { + val original = spark.conf.get(SQLConf.WAREHOUSE_PATH) + try { + // to get the default value, always unset it + spark.conf.unset(SQLConf.WAREHOUSE_PATH.key) + assert(spark.sessionState.conf.warehousePath + === s"file:${System.getProperty("user.dir")}/spark-warehouse") + } finally { + sql(s"set ${SQLConf.WAREHOUSE_PATH}=$original") + } + } + } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org