Github user srowen commented on a diff in the pull request:
https://github.com/apache/spark/pull/16344#discussion_r93215941
--- Diff:
mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala
---
@@ -592,6 +629,59 @@ object GeneralizedLinearRegression extends
DefaultParamsReadable[GeneralizedLine
}
/**
+ * Tweedie exponential family distribution.
+ * The default link for the Tweedie family is the log link.
+ */
+ private[regression] object Tweedie extends Family("tweedie") {
+
+ val defaultLink: Link = Log
+
+ var variancePower: Double = 1.5
+
+ override def initialize(y: Double, weight: Double): Double = {
+ if (variancePower > 1.0 && variancePower < 2.0) {
+ require(y >= 0.0, "The response variable of the specified Tweedie
distribution " +
+ s"should be non-negative, but got $y")
+ math.max(y, 0.1)
+ } else {
+ require(y > 0.0, "The response variable of the specified Tweedie
distribution " +
+ s"should be non-negative, but got $y")
+ y
+ }
+ }
+
+ override def variance(mu: Double): Double = math.pow(mu, variancePower)
+
+ private def yp(y: Double, mu: Double, p: Double): Double = {
+ (math.pow(y, p) - math.pow(mu, p)) / p
+ }
+
+ // Force y >= 0.1 for deviance to work for (1 - variancePower). see
tweedie()$dev.resid
+ override def deviance(y: Double, mu: Double, weight: Double): Double =
{
+ 2.0 * weight *
+ (y * yp(math.max(y, 0.1), mu, 1.0 - variancePower) - yp(y, mu, 2.0
- variancePower))
+ }
+
+ // This depends on the density of the tweedie distribution. Not yet
implemented.
+ override def aic(
+ predictions: RDD[(Double, Double, Double)],
+ deviance: Double,
+ numInstances: Double,
+ weightSum: Double): Double = {
+ 0.0
+ }
+
+ override def project(mu: Double): Double = {
+ if (mu < epsilon) {
+ epsilon
+ } else if (mu.isInfinity) {
+ Double.MaxValue
--- End diff --
Out of curiosity is this meaningful to "cap" at Double.MaxValue? By the
time you get there a lot of stuff is going to be infinite or not meaningful.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastruct...@apache.org or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
