Repository: spark Updated Branches: refs/heads/master 16612638f -> ae4ea5fe2
[SPARK-21524][ML] unit test fix: ValidatorParamsSuiteHelpers generates wrong temp files ## What changes were proposed in this pull request? jira: https://issues.apache.org/jira/browse/SPARK-21524 ValidatorParamsSuiteHelpers.testFileMove() is generating temp dir in the wrong place and does not delete them. ValidatorParamsSuiteHelpers.testFileMove() is invoked by TrainValidationSplitSuite and crossValidatorSuite. Currently it uses `tempDir` from `TempDirectory`, which unfortunately is never initialized since the `boforeAll()` of `ValidatorParamsSuiteHelpers` is never invoked. In my system, it leaves some temp directories in the assembly folder each time I run the TrainValidationSplitSuite and crossValidatorSuite. ## How was this patch tested? unit test fix Author: Yuhao Yang <yuhao.y...@intel.com> Closes #18728 from hhbyyh/tempDirFix. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ae4ea5fe Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ae4ea5fe Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ae4ea5fe Branch: refs/heads/master Commit: ae4ea5fe253e9083e807bdc769e829f3f37265b4 Parents: 1661263 Author: Yuhao Yang <yuhao.y...@intel.com> Authored: Wed Jul 26 10:37:48 2017 +0100 Committer: Sean Owen <so...@cloudera.com> Committed: Wed Jul 26 10:37:48 2017 +0100 ---------------------------------------------------------------------- .../org/apache/spark/ml/tuning/CrossValidatorSuite.scala | 2 +- .../apache/spark/ml/tuning/TrainValidationSplitSuite.scala | 2 +- .../spark/ml/tuning/ValidatorParamsSuiteHelpers.scala | 9 +++++---- 3 files changed, 7 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/ae4ea5fe/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala ---------------------------------------------------------------------- diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala index 2791ea7..dc6043e 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala @@ -222,7 +222,7 @@ class CrossValidatorSuite .setNumFolds(20) .setEstimatorParamMaps(paramMaps) - ValidatorParamsSuiteHelpers.testFileMove(cv) + ValidatorParamsSuiteHelpers.testFileMove(cv, tempDir) } test("read/write: CrossValidator with complex estimator") { http://git-wip-us.apache.org/repos/asf/spark/blob/ae4ea5fe/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala ---------------------------------------------------------------------- diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala index 71a1776..7c97865 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala @@ -209,7 +209,7 @@ class TrainValidationSplitSuite .setEstimatorParamMaps(paramMaps) .setSeed(42L) - ValidatorParamsSuiteHelpers.testFileMove(tvs) + ValidatorParamsSuiteHelpers.testFileMove(tvs, tempDir) } test("read/write: TrainValidationSplitModel") { http://git-wip-us.apache.org/repos/asf/spark/blob/ae4ea5fe/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala ---------------------------------------------------------------------- diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala index 1df673c..eae1f5a 100644 --- a/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala +++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala @@ -20,11 +20,12 @@ package org.apache.spark.ml.tuning import java.io.File import java.nio.file.{Files, StandardCopyOption} -import org.apache.spark.SparkFunSuite +import org.scalatest.Assertions + import org.apache.spark.ml.param.{ParamMap, ParamPair, Params} -import org.apache.spark.ml.util.{DefaultReadWriteTest, Identifiable, MLReader, MLWritable} +import org.apache.spark.ml.util.{Identifiable, MLReader, MLWritable} -object ValidatorParamsSuiteHelpers extends SparkFunSuite with DefaultReadWriteTest { +object ValidatorParamsSuiteHelpers extends Assertions { /** * Assert sequences of estimatorParamMaps are identical. * If the values for a parameter are not directly comparable with === @@ -62,7 +63,7 @@ object ValidatorParamsSuiteHelpers extends SparkFunSuite with DefaultReadWriteTe * the path of the estimator so that if the parent directory changes, loading the * model still works. */ - def testFileMove[T <: Params with MLWritable](instance: T): Unit = { + def testFileMove[T <: Params with MLWritable](instance: T, tempDir: File): Unit = { val uid = instance.uid val subdirName = Identifiable.randomUID("test") --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org