Github user jkbradley commented on a diff in the pull request:
https://github.com/apache/spark/pull/18281#discussion_r122524554
--- Diff:
mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala ---
@@ -101,6 +101,37 @@ class OneVsRestSuite extends SparkFunSuite with
MLlibTestSparkContext with Defau
assert(expectedMetrics.confusionMatrix ~== ovaMetrics.confusionMatrix
absTol 400)
}
+ test("one-vs-rest: tuning parallelism does not change output") {
+ val numClasses = 3
+ val ovaPar2 = new OneVsRest()
+ .setClassifier(new LogisticRegression)
+ .setParallelism(2)
+
+ val ovaModelPar2 = ovaPar2.fit(dataset)
+
+ val transformedDatasetPar2 = ovaModelPar2.transform(dataset)
+
+ val ovaResultsPar2 = transformedDatasetPar2.select("prediction",
"label").rdd.map {
+ row => (row.getDouble(0), row.getDouble(1))
+ }
+
+ val ovaPar4 = new OneVsRest()
+ .setClassifier(new LogisticRegression)
+ .setParallelism(4)
+
+ val ovaModelPar4 = ovaPar4.fit(dataset)
+
+ val transformedDatasetPar4 = ovaModelPar4.transform(dataset)
+
+ val ovaResultsPar4 = transformedDatasetPar4.select("prediction",
"label").rdd.map {
+ row => (row.getDouble(0), row.getDouble(1))
+ }
+
+ val metricsPar2 = new MulticlassMetrics(ovaResultsPar2)
+ val metricsPar4 = new MulticlassMetrics(ovaResultsPar4)
+ assert(metricsPar2.confusionMatrix ~== metricsPar4.confusionMatrix
absTol 400)
--- End diff --
+1
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
