Github user jkbradley commented on a diff in the pull request:
https://github.com/apache/spark/pull/19122#discussion_r140375849
--- Diff: python/pyspark/ml/tests.py ---
@@ -986,6 +1007,25 @@ def test_save_load_simple_estimator(self):
loadedModel = TrainValidationSplitModel.load(tvsModelPath)
self.assertEqual(loadedModel.bestModel.uid, tvsModel.bestModel.uid)
+ def test_parallel_evaluation(self):
+ dataset = self.spark.createDataFrame(
+ [(Vectors.dense([0.0]), 0.0),
+ (Vectors.dense([0.4]), 1.0),
+ (Vectors.dense([0.5]), 0.0),
+ (Vectors.dense([0.6]), 1.0),
+ (Vectors.dense([1.0]), 1.0)] * 10,
+ ["features", "label"])
+ lr = LogisticRegression()
+ grid = ParamGridBuilder().addGrid(lr.maxIter, [0, 1]).build()
+ evaluator = BinaryClassificationEvaluator()
+ tvs = TrainValidationSplit(estimator=lr, estimatorParamMaps=grid,
evaluator=evaluator)
+ tvs.setParallelism(1)
+ tvsSerialModel = tvs.fit(dataset)
+ tvs.setParallelism(2)
+ tvsParallelModel = tvs.fit(dataset)
+ self.assertEqual(sorted(tvsSerialModel.validationMetrics),
--- End diff --
ditto: don't sort the metrics
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
