spark git commit: [MINOR] Correct validateAndTransformSchema in GaussianMixture and AFTSurvivalRegression
Repository: spark
Updated Branches:
refs/heads/branch-2.1 6f366fbbf -> 2394ae235
[MINOR] Correct validateAndTransformSchema in GaussianMixture and
AFTSurvivalRegression
## What changes were proposed in this pull request?
The line SchemaUtils.appendColumn(schema, $(predictionCol), IntegerType) did
not modify the variable schema, hence only the last line had any effect. A
temporary variable is used to correctly append the two columns predictionCol
and probabilityCol.
## How was this patch tested?
Manually.
Please review http://spark.apache.org/contributing.html before opening a pull
request.
Author: Cédric Pelvet
Closes #18980 from sharp-pixel/master.
(cherry picked from commit 73e04ecc4f29a0fe51687ed1337c61840c976f89)
Signed-off-by: Sean Owen
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/2394ae23
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/2394ae23
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/2394ae23
Branch: refs/heads/branch-2.1
Commit: 2394ae23530259c6de398e57c4e625a68eda842a
Parents: 6f366fb
Author: Cédric Pelvet
Authored: Sun Aug 20 11:05:54 2017 +0100
Committer: Sean Owen
Committed: Sun Aug 20 11:06:14 2017 +0100
--
.../org/apache/spark/ml/clustering/GaussianMixture.scala | 4 ++--
.../apache/spark/ml/regression/AFTSurvivalRegression.scala | 8 +---
2 files changed, 7 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/2394ae23/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
--
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index ac56845..82aadea 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -62,8 +62,8 @@ private[clustering] trait GaussianMixtureParams extends
Params with HasMaxIter w
*/
protected def validateAndTransformSchema(schema: StructType): StructType = {
SchemaUtils.checkColumnType(schema, $(featuresCol), new VectorUDT)
-SchemaUtils.appendColumn(schema, $(predictionCol), IntegerType)
-SchemaUtils.appendColumn(schema, $(probabilityCol), new VectorUDT)
+val schemaWithPredictionCol = SchemaUtils.appendColumn(schema,
$(predictionCol), IntegerType)
+SchemaUtils.appendColumn(schemaWithPredictionCol, $(probabilityCol), new
VectorUDT)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/2394ae23/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
--
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
index af68e7b..fd74921 100644
---
a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
+++
b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
@@ -109,10 +109,12 @@ private[regression] trait AFTSurvivalRegressionParams
extends Params
SchemaUtils.checkColumnType(schema, $(censorCol), DoubleType)
SchemaUtils.checkNumericType(schema, $(labelCol))
}
-if (hasQuantilesCol) {
+
+val schemaWithQuantilesCol = if (hasQuantilesCol) {
SchemaUtils.appendColumn(schema, $(quantilesCol), new VectorUDT)
-}
-SchemaUtils.appendColumn(schema, $(predictionCol), DoubleType)
+} else schema
+
+SchemaUtils.appendColumn(schemaWithQuantilesCol, $(predictionCol),
DoubleType)
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [MINOR] Correct validateAndTransformSchema in GaussianMixture and AFTSurvivalRegression
Repository: spark
Updated Branches:
refs/heads/master 72b738d8d -> 73e04ecc4
[MINOR] Correct validateAndTransformSchema in GaussianMixture and
AFTSurvivalRegression
## What changes were proposed in this pull request?
The line SchemaUtils.appendColumn(schema, $(predictionCol), IntegerType) did
not modify the variable schema, hence only the last line had any effect. A
temporary variable is used to correctly append the two columns predictionCol
and probabilityCol.
## How was this patch tested?
Manually.
Please review http://spark.apache.org/contributing.html before opening a pull
request.
Author: Cédric Pelvet
Closes #18980 from sharp-pixel/master.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/73e04ecc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/73e04ecc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/73e04ecc
Branch: refs/heads/master
Commit: 73e04ecc4f29a0fe51687ed1337c61840c976f89
Parents: 72b738d
Author: Cédric Pelvet
Authored: Sun Aug 20 11:05:54 2017 +0100
Committer: Sean Owen
Committed: Sun Aug 20 11:05:54 2017 +0100
--
.../org/apache/spark/ml/clustering/GaussianMixture.scala | 4 ++--
.../apache/spark/ml/regression/AFTSurvivalRegression.scala | 8 +---
2 files changed, 7 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/73e04ecc/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
--
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index 5259ee4..f19ad7a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -64,8 +64,8 @@ private[clustering] trait GaussianMixtureParams extends
Params with HasMaxIter w
*/
protected def validateAndTransformSchema(schema: StructType): StructType = {
SchemaUtils.checkColumnType(schema, $(featuresCol), new VectorUDT)
-SchemaUtils.appendColumn(schema, $(predictionCol), IntegerType)
-SchemaUtils.appendColumn(schema, $(probabilityCol), new VectorUDT)
+val schemaWithPredictionCol = SchemaUtils.appendColumn(schema,
$(predictionCol), IntegerType)
+SchemaUtils.appendColumn(schemaWithPredictionCol, $(probabilityCol), new
VectorUDT)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/73e04ecc/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
--
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
index 0891994..16821f3 100644
---
a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
+++
b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
@@ -109,10 +109,12 @@ private[regression] trait AFTSurvivalRegressionParams
extends Params
SchemaUtils.checkNumericType(schema, $(censorCol))
SchemaUtils.checkNumericType(schema, $(labelCol))
}
-if (hasQuantilesCol) {
+
+val schemaWithQuantilesCol = if (hasQuantilesCol) {
SchemaUtils.appendColumn(schema, $(quantilesCol), new VectorUDT)
-}
-SchemaUtils.appendColumn(schema, $(predictionCol), DoubleType)
+} else schema
+
+SchemaUtils.appendColumn(schemaWithQuantilesCol, $(predictionCol),
DoubleType)
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [MINOR] Correct validateAndTransformSchema in GaussianMixture and AFTSurvivalRegression
Repository: spark
Updated Branches:
refs/heads/branch-2.2 fdea642db -> 6c2a38a38
[MINOR] Correct validateAndTransformSchema in GaussianMixture and
AFTSurvivalRegression
## What changes were proposed in this pull request?
The line SchemaUtils.appendColumn(schema, $(predictionCol), IntegerType) did
not modify the variable schema, hence only the last line had any effect. A
temporary variable is used to correctly append the two columns predictionCol
and probabilityCol.
## How was this patch tested?
Manually.
Please review http://spark.apache.org/contributing.html before opening a pull
request.
Author: Cédric Pelvet
Closes #18980 from sharp-pixel/master.
(cherry picked from commit 73e04ecc4f29a0fe51687ed1337c61840c976f89)
Signed-off-by: Sean Owen
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/6c2a38a3
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/6c2a38a3
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/6c2a38a3
Branch: refs/heads/branch-2.2
Commit: 6c2a38a381f22029abd9ca4beab49b2473a13670
Parents: fdea642
Author: Cédric Pelvet
Authored: Sun Aug 20 11:05:54 2017 +0100
Committer: Sean Owen
Committed: Sun Aug 20 11:06:02 2017 +0100
--
.../org/apache/spark/ml/clustering/GaussianMixture.scala | 4 ++--
.../apache/spark/ml/regression/AFTSurvivalRegression.scala | 8 +---
2 files changed, 7 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/6c2a38a3/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
--
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index 5259ee4..f19ad7a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -64,8 +64,8 @@ private[clustering] trait GaussianMixtureParams extends
Params with HasMaxIter w
*/
protected def validateAndTransformSchema(schema: StructType): StructType = {
SchemaUtils.checkColumnType(schema, $(featuresCol), new VectorUDT)
-SchemaUtils.appendColumn(schema, $(predictionCol), IntegerType)
-SchemaUtils.appendColumn(schema, $(probabilityCol), new VectorUDT)
+val schemaWithPredictionCol = SchemaUtils.appendColumn(schema,
$(predictionCol), IntegerType)
+SchemaUtils.appendColumn(schemaWithPredictionCol, $(probabilityCol), new
VectorUDT)
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/6c2a38a3/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
--
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
index 0891994..16821f3 100644
---
a/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
+++
b/mllib/src/main/scala/org/apache/spark/ml/regression/AFTSurvivalRegression.scala
@@ -109,10 +109,12 @@ private[regression] trait AFTSurvivalRegressionParams
extends Params
SchemaUtils.checkNumericType(schema, $(censorCol))
SchemaUtils.checkNumericType(schema, $(labelCol))
}
-if (hasQuantilesCol) {
+
+val schemaWithQuantilesCol = if (hasQuantilesCol) {
SchemaUtils.appendColumn(schema, $(quantilesCol), new VectorUDT)
-}
-SchemaUtils.appendColumn(schema, $(predictionCol), DoubleType)
+} else schema
+
+SchemaUtils.appendColumn(schemaWithQuantilesCol, $(predictionCol),
DoubleType)
}
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-21773][BUILD][DOCS] Installs mkdocs if missing in the path in SQL documentation build
Repository: spark Updated Branches: refs/heads/master 73e04ecc4 -> 41e0eb71a [SPARK-21773][BUILD][DOCS] Installs mkdocs if missing in the path in SQL documentation build ## What changes were proposed in this pull request? This PR proposes to install `mkdocs` by `pip install` if missing in the path. Mainly to fix Jenkins's documentation build failure in `spark-master-docs`. See https://amplab.cs.berkeley.edu/jenkins/job/spark-master-docs/3580/console. It also adds `mkdocs` as requirements in `docs/README.md`. ## How was this patch tested? I manually ran `jekyll build` under `docs` directory after manually removing `mkdocs` via `pip uninstall mkdocs`. Also, tested this in the same way but on CentOS Linux release 7.3.1611 (Core) where I built Spark few times but never built documentation before and `mkdocs` is not installed. ``` ... Moving back into docs dir. Moving to SQL directory and building docs. Missing mkdocs in your path, trying to install mkdocs for SQL documentation generation. Collecting mkdocs Downloading mkdocs-0.16.3-py2.py3-none-any.whl (1.2MB) 100% |ââââââââââââââââââââââââââââââââ| 1.2MB 574kB/s Requirement already satisfied: PyYAML>=3.10 in /usr/lib64/python2.7/site-packages (from mkdocs) Collecting livereload>=2.5.1 (from mkdocs) Downloading livereload-2.5.1-py2-none-any.whl Collecting tornado>=4.1 (from mkdocs) Downloading tornado-4.5.1.tar.gz (483kB) 100% |ââââââââââââââââââââââââââââââââ| 491kB 1.4MB/s Collecting Markdown>=2.3.1 (from mkdocs) Downloading Markdown-2.6.9.tar.gz (271kB) 100% |ââââââââââââââââââââââââââââââââ| 276kB 2.4MB/s Collecting click>=3.3 (from mkdocs) Downloading click-6.7-py2.py3-none-any.whl (71kB) 100% |ââââââââââââââââââââââââââââââââ| 71kB 2.8MB/s Requirement already satisfied: Jinja2>=2.7.1 in /usr/lib/python2.7/site-packages (from mkdocs) Requirement already satisfied: six in /usr/lib/python2.7/site-packages (from livereload>=2.5.1->mkdocs) Requirement already satisfied: backports.ssl_match_hostname in /usr/lib/python2.7/site-packages (from tornado>=4.1->mkdocs) Collecting singledispatch (from tornado>=4.1->mkdocs) Downloading singledispatch-3.4.0.3-py2.py3-none-any.whl Collecting certifi (from tornado>=4.1->mkdocs) Downloading certifi-2017.7.27.1-py2.py3-none-any.whl (349kB) 100% |ââââââââââââââââââââââââââââââââ| 358kB 2.1MB/s Collecting backports_abc>=0.4 (from tornado>=4.1->mkdocs) Downloading backports_abc-0.5-py2.py3-none-any.whl Requirement already satisfied: MarkupSafe>=0.23 in /usr/lib/python2.7/site-packages (from Jinja2>=2.7.1->mkdocs) Building wheels for collected packages: tornado, Markdown Running setup.py bdist_wheel for tornado ... done Stored in directory: /root/.cache/pip/wheels/84/83/cd/6a04602633457269d161344755e6766d24307189b7a67ff4b7 Running setup.py bdist_wheel for Markdown ... done Stored in directory: /root/.cache/pip/wheels/bf/46/10/c93e17ae86ae3b3a919c7b39dad3b5ccf09aeb066419e5c1e5 Successfully built tornado Markdown Installing collected packages: singledispatch, certifi, backports-abc, tornado, livereload, Markdown, click, mkdocs Successfully installed Markdown-2.6.9 backports-abc-0.5 certifi-2017.7.27.1 click-6.7 livereload-2.5.1 mkdocs-0.16.3 singledispatch-3.4.0.3 tornado-4.5.1 Generating markdown files for SQL documentation. Generating HTML files for SQL documentation. INFO- Cleaning site directory INFO- Building documentation to directory: .../spark/sql/site Moving back into docs dir. Making directory api/sql cp -r ../sql/site/. api/sql Source: .../spark/docs Destination: .../spark/docs/_site Generating... done. Auto-regeneration: disabled. Use --watch to enable. ``` Author: hyukjinkwon Closes #18984 from HyukjinKwon/sql-doc-mkdocs. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/41e0eb71 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/41e0eb71 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/41e0eb71 Branch: refs/heads/master Commit: 41e0eb71a63140c9a44a7d2f32821f02abd62367 Parents: 73e04ec Author: hyukjinkwon Authored: Sun Aug 20 19:48:04 2017 +0900 Committer: hyukjinkwon Committed: Sun Aug 20 19:48:04 2017 +0900 -- docs/README.md | 2 +- sql/create-docs.sh | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/41e0eb71/docs/README.md -
spark git commit: [SPARK-21721][SQL][BACKPORT-2.1][FOLLOWUP] Clear FileSystem deleteOnExit cache when paths are successfully removed
Repository: spark
Updated Branches:
refs/heads/branch-2.1 2394ae235 -> 3d3be4dca
[SPARK-21721][SQL][BACKPORT-2.1][FOLLOWUP] Clear FileSystem deleteOnExit cache
when paths are successfully removed
## What changes were proposed in this pull request?
Fix a typo in test.
## How was this patch tested?
Jenkins tests.
Author: Liang-Chi Hsieh
Closes #19006 from viirya/SPARK-21721-backport-2.1-followup.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3d3be4dc
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3d3be4dc
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3d3be4dc
Branch: refs/heads/branch-2.1
Commit: 3d3be4dca48b7fab8b6754a5eb6ce671353f34a1
Parents: 2394ae2
Author: Liang-Chi Hsieh
Authored: Sun Aug 20 21:57:27 2017 +0800
Committer: Wenchen Fan
Committed: Sun Aug 20 21:57:27 2017 +0800
--
.../org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/3d3be4dc/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index 73ceaf8..b2c5fb9 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -2034,7 +2034,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils
with TestHiveSingleton {
}
test("SPARK-21721: Clear FileSystem deleterOnExit cache if path is
successfully removed") {
-withTable("test21721") {
+val table = "test21721"
+withTable(table) {
val deleteOnExitField =
classOf[FileSystem].getDeclaredField("deleteOnExit")
deleteOnExitField.setAccessible(true)
@@ -2042,10 +2043,10 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils
with TestHiveSingleton {
val setOfPath = deleteOnExitField.get(fs).asInstanceOf[Set[Path]]
val testData = sparkContext.parallelize(1 to 10).map(i => TestData(i,
i.toString)).toDF()
- sql("CREATE TABLE test21721 (key INT, value STRING)")
+ sql(s"CREATE TABLE $table (key INT, value STRING)")
val pathSizeToDeleteOnExit = setOfPath.size()
- (0 to 10).foreach(_ =>
testData.write.mode(SaveMode.Append).insertInto("test1"))
+ (0 to 10).foreach(_ =>
testData.write.mode(SaveMode.Append).insertInto(table))
assert(setOfPath.size() == pathSizeToDeleteOnExit)
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-21721][SQL][FOLLOWUP] Clear FileSystem deleteOnExit cache when paths are successfully removed
Repository: spark
Updated Branches:
refs/heads/branch-2.2 6c2a38a38 -> 0f640e96c
[SPARK-21721][SQL][FOLLOWUP] Clear FileSystem deleteOnExit cache when paths are
successfully removed
## What changes were proposed in this pull request?
Fix a typo in test.
## How was this patch tested?
Jenkins tests.
Author: Liang-Chi Hsieh
Closes #19005 from viirya/SPARK-21721-followup.
(cherry picked from commit 28a6cca7df900d13613b318c07acb97a5722d2b8)
Signed-off-by: Wenchen Fan
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0f640e96
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0f640e96
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0f640e96
Branch: refs/heads/branch-2.2
Commit: 0f640e96c9d0b0d95ac4bbcc84eaccefe7352f0f
Parents: 6c2a38a
Author: Liang-Chi Hsieh
Authored: Mon Aug 21 00:45:23 2017 +0800
Committer: Wenchen Fan
Committed: Mon Aug 21 00:45:39 2017 +0800
--
.../org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/0f640e96/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index dc79bfa..31b36f1 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -2017,7 +2017,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils
with TestHiveSingleton {
}
test("SPARK-21721: Clear FileSystem deleterOnExit cache if path is
successfully removed") {
-withTable("test21721") {
+val table = "test21721"
+withTable(table) {
val deleteOnExitField =
classOf[FileSystem].getDeclaredField("deleteOnExit")
deleteOnExitField.setAccessible(true)
@@ -2025,10 +2026,10 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils
with TestHiveSingleton {
val setOfPath = deleteOnExitField.get(fs).asInstanceOf[Set[Path]]
val testData = sparkContext.parallelize(1 to 10).map(i => TestData(i,
i.toString)).toDF()
- sql("CREATE TABLE test21721 (key INT, value STRING)")
+ sql(s"CREATE TABLE $table (key INT, value STRING)")
val pathSizeToDeleteOnExit = setOfPath.size()
- (0 to 10).foreach(_ =>
testData.write.mode(SaveMode.Append).insertInto("test1"))
+ (0 to 10).foreach(_ =>
testData.write.mode(SaveMode.Append).insertInto(table))
assert(setOfPath.size() == pathSizeToDeleteOnExit)
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
spark git commit: [SPARK-21721][SQL][FOLLOWUP] Clear FileSystem deleteOnExit cache when paths are successfully removed
Repository: spark
Updated Branches:
refs/heads/master 41e0eb71a -> 28a6cca7d
[SPARK-21721][SQL][FOLLOWUP] Clear FileSystem deleteOnExit cache when paths are
successfully removed
## What changes were proposed in this pull request?
Fix a typo in test.
## How was this patch tested?
Jenkins tests.
Author: Liang-Chi Hsieh
Closes #19005 from viirya/SPARK-21721-followup.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/28a6cca7
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/28a6cca7
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/28a6cca7
Branch: refs/heads/master
Commit: 28a6cca7df900d13613b318c07acb97a5722d2b8
Parents: 41e0eb7
Author: Liang-Chi Hsieh
Authored: Mon Aug 21 00:45:23 2017 +0800
Committer: Wenchen Fan
Committed: Mon Aug 21 00:45:23 2017 +0800
--
.../org/apache/spark/sql/hive/execution/SQLQuerySuite.scala | 7 ---
1 file changed, 4 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/spark/blob/28a6cca7/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
--
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
index ef3d9b2..d0e0d20 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/SQLQuerySuite.scala
@@ -2023,7 +2023,8 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils
with TestHiveSingleton {
}
test("SPARK-21721: Clear FileSystem deleterOnExit cache if path is
successfully removed") {
-withTable("test21721") {
+val table = "test21721"
+withTable(table) {
val deleteOnExitField =
classOf[FileSystem].getDeclaredField("deleteOnExit")
deleteOnExitField.setAccessible(true)
@@ -2031,10 +2032,10 @@ class SQLQuerySuite extends QueryTest with SQLTestUtils
with TestHiveSingleton {
val setOfPath = deleteOnExitField.get(fs).asInstanceOf[Set[Path]]
val testData = sparkContext.parallelize(1 to 10).map(i => TestData(i,
i.toString)).toDF()
- sql("CREATE TABLE test21721 (key INT, value STRING)")
+ sql(s"CREATE TABLE $table (key INT, value STRING)")
val pathSizeToDeleteOnExit = setOfPath.size()
- (0 to 10).foreach(_ =>
testData.write.mode(SaveMode.Append).insertInto("test1"))
+ (0 to 10).foreach(_ =>
testData.write.mode(SaveMode.Append).insertInto(table))
assert(setOfPath.size() == pathSizeToDeleteOnExit)
}
-
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
