Github user shivaram commented on a diff in the pull request:
https://github.com/apache/spark/pull/3637#discussion_r23880618
--- Diff:
mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
---
@@ -80,69 +50,157 @@ class LogisticRegression extends
Estimator[LogisticRegressionModel] with Logisti
def setRegParam(value: Double): this.type = set(regParam, value)
def setMaxIter(value: Int): this.type = set(maxIter, value)
- def setLabelCol(value: String): this.type = set(labelCol, value)
def setThreshold(value: Double): this.type = set(threshold, value)
- def setFeaturesCol(value: String): this.type = set(featuresCol, value)
- def setScoreCol(value: String): this.type = set(scoreCol, value)
- def setPredictionCol(value: String): this.type = set(predictionCol,
value)
override def fit(dataset: SchemaRDD, paramMap: ParamMap):
LogisticRegressionModel = {
+ // Check schema
transformSchema(dataset.schema, paramMap, logging = true)
- import dataset.sqlContext._
+
+ // Extract columns from data. If dataset is persisted, do not persist
oldDataset.
+ val oldDataset = extractLabeledPoints(dataset, paramMap)
val map = this.paramMap ++ paramMap
- val instances = dataset.select(map(labelCol).attr,
map(featuresCol).attr)
- .map { case Row(label: Double, features: Vector) =>
- LabeledPoint(label, features)
- }.persist(StorageLevel.MEMORY_AND_DISK)
+ val handlePersistence = dataset.getStorageLevel == StorageLevel.NONE
+ if (handlePersistence) {
+ oldDataset.persist(StorageLevel.MEMORY_AND_DISK)
+ }
+
+ // Train model
val lr = new LogisticRegressionWithLBFGS
lr.optimizer
.setRegParam(map(regParam))
.setNumIterations(map(maxIter))
- val lrm = new LogisticRegressionModel(this, map,
lr.run(instances).weights)
- instances.unpersist()
+ val oldModel = lr.run(oldDataset)
+ val lrm = new LogisticRegressionModel(this, map, oldModel.weights,
oldModel.intercept)
+
+ if (handlePersistence) {
+ oldDataset.unpersist()
+ }
+
// copy model params
Params.inheritValues(map, this, lrm)
lrm
}
- private[ml] override def transformSchema(schema: StructType, paramMap:
ParamMap): StructType = {
- validateAndTransformSchema(schema, paramMap, fitting = true)
- }
+ override protected def featuresDataType: DataType = new VectorUDT
}
+
/**
* :: AlphaComponent ::
+ *
* Model produced by [[LogisticRegression]].
*/
@AlphaComponent
class LogisticRegressionModel private[ml] (
override val parent: LogisticRegression,
override val fittingParamMap: ParamMap,
- weights: Vector)
- extends Model[LogisticRegressionModel] with LogisticRegressionParams {
+ val weights: Vector,
+ val intercept: Double)
+ extends ProbabilisticClassificationModel[Vector, LogisticRegressionModel]
+ with LogisticRegressionParams {
+
+ setThreshold(0.5)
def setThreshold(value: Double): this.type = set(threshold, value)
- def setFeaturesCol(value: String): this.type = set(featuresCol, value)
- def setScoreCol(value: String): this.type = set(scoreCol, value)
- def setPredictionCol(value: String): this.type = set(predictionCol,
value)
- private[ml] override def transformSchema(schema: StructType, paramMap:
ParamMap): StructType = {
- validateAndTransformSchema(schema, paramMap, fitting = false)
+ private val margin: Vector => Double = (features) => {
+ BLAS.dot(features, weights) + intercept
+ }
+
+ private val score: Vector => Double = (features) => {
+ val m = margin(features)
+ 1.0 / (1.0 + math.exp(-m))
}
override def transform(dataset: SchemaRDD, paramMap: ParamMap):
SchemaRDD = {
+ // Check schema
transformSchema(dataset.schema, paramMap, logging = true)
+
import dataset.sqlContext._
val map = this.paramMap ++ paramMap
- val score: Vector => Double = (v) => {
- val margin = BLAS.dot(v, weights)
- 1.0 / (1.0 + math.exp(-margin))
+
+ // Output selected columns only.
+ // This is a bit complicated since it tries to avoid repeated
computation.
--- End diff --
Is this going to be a part of this PR ? I'm also okay with adding a `TODO`
here to add the functions later
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
