Github user MrBago commented on a diff in the pull request:
https://github.com/apache/spark/pull/19527#discussion_r145531800
--- Diff:
mllib/src/main/scala/org/apache/spark/ml/feature/OneHotEncoderEstimator.scala
---
@@ -0,0 +1,439 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.ml.feature
+
+import org.apache.hadoop.fs.Path
+
+import org.apache.spark.SparkException
+import org.apache.spark.annotation.Since
+import org.apache.spark.ml.{Estimator, Model, Transformer}
+import org.apache.spark.ml.attribute._
+import org.apache.spark.ml.linalg.Vectors
+import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared.{HasHandleInvalid, HasInputCol,
HasInputCols, HasOutputCol, HasOutputCols}
+import org.apache.spark.ml.util._
+import org.apache.spark.sql.{DataFrame, Dataset}
+import org.apache.spark.sql.expressions.UserDefinedFunction
+import org.apache.spark.sql.functions.{col, udf}
+import org.apache.spark.sql.types.{DoubleType, NumericType, StructField,
StructType}
+
+/** Private trait for params for OneHotEncoderEstimator and
OneHotEncoderModel */
+private[ml] trait OneHotEncoderParams extends Params with HasHandleInvalid
+ with HasInputCols with HasOutputCols {
+
+ /**
+ * Param for how to handle invalid data.
+ * Options are 'skip' (filter out rows with invalid data) or 'error'
(throw an error).
+ * Default: "error"
+ * @group param
+ */
+ @Since("2.3.0")
+ override val handleInvalid: Param[String] = new Param[String](this,
"handleInvalid",
+ "How to handle invalid data " +
+ "Options are 'skip' (filter out rows with invalid data) or error
(throw an error).",
+
ParamValidators.inArray(OneHotEncoderEstimator.supportedHandleInvalids))
+
+ setDefault(handleInvalid, OneHotEncoderEstimator.ERROR_INVALID)
+
+ /**
+ * Whether to drop the last category in the encoded vector (default:
true)
+ * @group param
+ */
+ @Since("2.3.0")
+ final val dropLast: BooleanParam =
+ new BooleanParam(this, "dropLast", "whether to drop the last category")
+ setDefault(dropLast -> true)
+
+ /** @group getParam */
+ @Since("2.3.0")
+ def getDropLast: Boolean = $(dropLast)
+}
+
+/**
+ * A one-hot encoder that maps a column of category indices to a column of
binary vectors, with
+ * at most a single one-value per row that indicates the input category
index.
+ * For example with 5 categories, an input value of 2.0 would map to an
output vector of
+ * `[0.0, 0.0, 1.0, 0.0]`.
+ * The last category is not included by default (configurable via
`dropLast`),
+ * because it makes the vector entries sum up to one, and hence linearly
dependent.
+ * So an input value of 4.0 maps to `[0.0, 0.0, 0.0, 0.0]`.
+ *
+ * @note This is different from scikit-learn's OneHotEncoder, which keeps
all categories.
+ * The output vectors are sparse.
+ *
+ * @see `StringIndexer` for converting categorical values into category
indices
+ */
+@Since("2.3.0")
+class OneHotEncoderEstimator @Since("2.3.0") (@Since("2.3.0") override val
uid: String)
+ extends Estimator[OneHotEncoderModel] with OneHotEncoderParams with
DefaultParamsWritable {
+
+ @Since("2.3.0")
+ def this() = this(Identifiable.randomUID("oneHotEncoder"))
+
+ /** @group setParam */
+ @Since("2.3.0")
+ def setInputCols(values: Array[String]): this.type = set(inputCols,
values)
+
+ /** @group setParam */
+ @Since("2.3.0")
+ def setOutputCols(values: Array[String]): this.type = set(outputCols,
values)
+
+ /** @group setParam */
+ @Since("2.3.0")
+ def setDropLast(value: Boolean): this.type = set(dropLast, value)
+
+ /** @group setParam */
+ @Since("2.3.0")
+ def setHandleInvalid(value: String): this.type = set(handleInvalid,
value)
+
+ @Since("2.3.0")
+ override def transformSchema(schema: StructType): StructType = {
+ val inputColNames = $(inputCols)
+ val outputColNames = $(outputCols)
+ val inputFields = schema.fields
+
+ require(inputColNames.length == outputColNames.length,
+ s"The number of input columns ${inputColNames.length} must be the
same as the number of " +
+ s"output columns ${outputColNames.length}.")
+
+ val outputFields = inputColNames.zip(outputColNames).map { case
(inputColName, outputColName) =>
+
+ require(schema(inputColName).dataType.isInstanceOf[NumericType],
+ s"Input column must be of type NumericType but got
${schema(inputColName).dataType}")
+ require(!inputFields.exists(_.name == outputColName),
+ s"Output column $outputColName already exists.")
+
+ OneHotEncoderCommon.transformOutputColumnSchema(
+ schema(inputColName), $(dropLast), outputColName)
+ }
+ StructType(inputFields ++ outputFields)
+ }
+
+ @Since("2.3.0")
+ override def fit(dataset: Dataset[_]): OneHotEncoderModel = {
+ val transformedSchema = transformSchema(dataset.schema)
+
+ val categorySizes = $(outputCols).zipWithIndex.map { case
(outputColName, idx) =>
+ val outputAttrGroupFromSchema = AttributeGroup.fromStructField(
+ transformedSchema(outputColName))
+
+ val outputAttrGroup = if (outputAttrGroupFromSchema.size < 0) {
+ OneHotEncoderCommon.getOutputAttrGroupFromData(
+ dataset, $(dropLast), $(inputCols)(idx), outputColName)
+ } else {
+ outputAttrGroupFromSchema
+ }
+
+ outputAttrGroup.size
+ }
+
+ val model = new OneHotEncoderModel(uid, categorySizes).setParent(this)
+ copyValues(model)
+ }
+
+ @Since("2.3.0")
+ override def copy(extra: ParamMap): OneHotEncoderEstimator =
defaultCopy(extra)
+}
+
+@Since("2.3.0")
+object OneHotEncoderEstimator extends
DefaultParamsReadable[OneHotEncoderEstimator] {
+
+ private[feature] val SKIP_INVALID: String = "skip"
+ private[feature] val ERROR_INVALID: String = "error"
+ private[feature] val supportedHandleInvalids: Array[String] =
Array(SKIP_INVALID, ERROR_INVALID)
+
+ @Since("2.3.0")
+ override def load(path: String): OneHotEncoderEstimator =
super.load(path)
+}
+
+@Since("2.3.0")
+class OneHotEncoderModel private[ml] (
+ @Since("2.3.0") override val uid: String,
+ @Since("2.3.0") val categorySizes: Array[Int])
+ extends Model[OneHotEncoderModel] with OneHotEncoderParams with
MLWritable {
+
+ import OneHotEncoderModel._
+
+ private def encoders: Array[UserDefinedFunction] = {
+ val oneValue = Array(1.0)
+ val emptyValues = Array.empty[Double]
+ val emptyIndices = Array.empty[Int]
+ val dropLast = getDropLast
+ val handleInvalid = getHandleInvalid
+
+ categorySizes.map { size =>
+ udf { label: Double =>
+ if (label < size) {
+ Vectors.sparse(size, Array(label.toInt), oneValue)
+ } else if (label == size && dropLast) {
+ Vectors.sparse(size, emptyIndices, emptyValues)
+ } else {
+ if (handleInvalid == OneHotEncoderEstimator.ERROR_INVALID) {
+ throw new SparkException(s"Unseen value: $label. To handle
unseen values, " +
+ s"set Param handleInvalid to
${OneHotEncoderEstimator.SKIP_INVALID}.")
+ } else {
+ Vectors.sparse(size, emptyIndices, emptyValues)
--- End diff --
This behaviour seems more like "keep" in StringIndexer than "skip". Skip
filters out rows with invalid values and returns a dataframe with number of
rows <= the number of rows in the input. "keep" maps all invalid values to a
single, known value. Should we rename this to `KEEP_INVALID`?
https://spark.apache.org/docs/2.2.0/ml-features.html#stringindexer
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
