spark git commit: [SPARK-15721][ML] Make DefaultParamsReadable, DefaultParamsWritable public
Repository: spark Updated Branches: refs/heads/branch-2.0 e38ff70e6 -> d07bce49f [SPARK-15721][ML] Make DefaultParamsReadable, DefaultParamsWritable public ## What changes were proposed in this pull request? Made DefaultParamsReadable, DefaultParamsWritable public. Also added relevant doc and annotations. Added UnaryTransformerExample to demonstrate use of UnaryTransformer and DefaultParamsReadable,Writable. ## How was this patch tested? Wrote example making use of the now-public APIs. Compiled and ran locally Author: Joseph K. Bradley Closes #13461 from jkbradley/defaultparamswritable. (cherry picked from commit 4c74ee8d8e1c3139d3d322ae68977f2ab53295df) Signed-off-by: Joseph K. Bradley Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d07bce49 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d07bce49 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d07bce49 Branch: refs/heads/branch-2.0 Commit: d07bce49fc77aff25330920cc55b7079a3a2995c Parents: e38ff70 Author: Joseph K. Bradley Authored: Mon Jun 6 09:49:45 2016 -0700 Committer: Joseph K. Bradley Committed: Mon Jun 6 09:49:56 2016 -0700 -- .../examples/ml/UnaryTransformerExample.scala | 122 +++ .../org/apache/spark/ml/util/ReadWrite.scala| 44 ++- 2 files changed, 163 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/d07bce49/examples/src/main/scala/org/apache/spark/examples/ml/UnaryTransformerExample.scala -- diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/UnaryTransformerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/UnaryTransformerExample.scala new file mode 100644 index 000..13c72f8 --- /dev/null +++ b/examples/src/main/scala/org/apache/spark/examples/ml/UnaryTransformerExample.scala @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// scalastyle:off println +package org.apache.spark.examples.ml + +// $example on$ +import org.apache.spark.ml.UnaryTransformer +import org.apache.spark.ml.param.DoubleParam +import org.apache.spark.ml.util.{DefaultParamsReadable, DefaultParamsWritable, Identifiable} +import org.apache.spark.sql.functions.col +// $example off$ +import org.apache.spark.sql.SparkSession +// $example on$ +import org.apache.spark.sql.types.{DataType, DataTypes} +import org.apache.spark.util.Utils +// $example off$ + +/** + * An example demonstrating creating a custom [[org.apache.spark.ml.Transformer]] using + * the [[UnaryTransformer]] abstraction. + * + * Run with + * {{{ + * bin/run-example ml.UnaryTransformerExample + * }}} + */ +object UnaryTransformerExample { + + // $example on$ + /** + * Simple Transformer which adds a constant value to input Doubles. + * + * [[UnaryTransformer]] can be used to create a stage usable within Pipelines. + * It defines parameters for specifying input and output columns: + * [[UnaryTransformer.inputCol]] and [[UnaryTransformer.outputCol]]. + * It can optionally handle schema validation. + * + * [[DefaultParamsWritable]] provides a default implementation for persisting instances + * of this Transformer. + */ + class MyTransformer(override val uid: String) +extends UnaryTransformer[Double, Double, MyTransformer] with DefaultParamsWritable { + +final val shift: DoubleParam = new DoubleParam(this, "shift", "Value added to input") + +def getShift: Double = $(shift) + +def setShift(value: Double): this.type = set(shift, value) + +def this() = this(Identifiable.randomUID("myT")) + +override protected def createTransformFunc: Double => Double = (input: Double) => { + input + $(shift) +} + +override protected def outputDataType: DataType = DataTypes.DoubleType + +override protected def validateInputType(inputType: DataType): Unit = { + require(inputType == DataTypes.DoubleType, s"Bad input type: $inputType. Requires Double.") +} + } + + /** + * C
spark git commit: [SPARK-15721][ML] Make DefaultParamsReadable, DefaultParamsWritable public
Repository: spark Updated Branches: refs/heads/master fa4bc8ea8 -> 4c74ee8d8 [SPARK-15721][ML] Make DefaultParamsReadable, DefaultParamsWritable public ## What changes were proposed in this pull request? Made DefaultParamsReadable, DefaultParamsWritable public. Also added relevant doc and annotations. Added UnaryTransformerExample to demonstrate use of UnaryTransformer and DefaultParamsReadable,Writable. ## How was this patch tested? Wrote example making use of the now-public APIs. Compiled and ran locally Author: Joseph K. Bradley Closes #13461 from jkbradley/defaultparamswritable. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4c74ee8d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4c74ee8d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4c74ee8d Branch: refs/heads/master Commit: 4c74ee8d8e1c3139d3d322ae68977f2ab53295df Parents: fa4bc8e Author: Joseph K. Bradley Authored: Mon Jun 6 09:49:45 2016 -0700 Committer: Joseph K. Bradley Committed: Mon Jun 6 09:49:45 2016 -0700 -- .../examples/ml/UnaryTransformerExample.scala | 122 +++ .../org/apache/spark/ml/util/ReadWrite.scala| 44 ++- 2 files changed, 163 insertions(+), 3 deletions(-) -- http://git-wip-us.apache.org/repos/asf/spark/blob/4c74ee8d/examples/src/main/scala/org/apache/spark/examples/ml/UnaryTransformerExample.scala -- diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/UnaryTransformerExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/UnaryTransformerExample.scala new file mode 100644 index 000..13c72f8 --- /dev/null +++ b/examples/src/main/scala/org/apache/spark/examples/ml/UnaryTransformerExample.scala @@ -0,0 +1,122 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + *http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// scalastyle:off println +package org.apache.spark.examples.ml + +// $example on$ +import org.apache.spark.ml.UnaryTransformer +import org.apache.spark.ml.param.DoubleParam +import org.apache.spark.ml.util.{DefaultParamsReadable, DefaultParamsWritable, Identifiable} +import org.apache.spark.sql.functions.col +// $example off$ +import org.apache.spark.sql.SparkSession +// $example on$ +import org.apache.spark.sql.types.{DataType, DataTypes} +import org.apache.spark.util.Utils +// $example off$ + +/** + * An example demonstrating creating a custom [[org.apache.spark.ml.Transformer]] using + * the [[UnaryTransformer]] abstraction. + * + * Run with + * {{{ + * bin/run-example ml.UnaryTransformerExample + * }}} + */ +object UnaryTransformerExample { + + // $example on$ + /** + * Simple Transformer which adds a constant value to input Doubles. + * + * [[UnaryTransformer]] can be used to create a stage usable within Pipelines. + * It defines parameters for specifying input and output columns: + * [[UnaryTransformer.inputCol]] and [[UnaryTransformer.outputCol]]. + * It can optionally handle schema validation. + * + * [[DefaultParamsWritable]] provides a default implementation for persisting instances + * of this Transformer. + */ + class MyTransformer(override val uid: String) +extends UnaryTransformer[Double, Double, MyTransformer] with DefaultParamsWritable { + +final val shift: DoubleParam = new DoubleParam(this, "shift", "Value added to input") + +def getShift: Double = $(shift) + +def setShift(value: Double): this.type = set(shift, value) + +def this() = this(Identifiable.randomUID("myT")) + +override protected def createTransformFunc: Double => Double = (input: Double) => { + input + $(shift) +} + +override protected def outputDataType: DataType = DataTypes.DoubleType + +override protected def validateInputType(inputType: DataType): Unit = { + require(inputType == DataTypes.DoubleType, s"Bad input type: $inputType. Requires Double.") +} + } + + /** + * Companion object for our simple Transformer. + * + * [[DefaultParamsReadable]] provides a default implementat