Mailing-List: contact reviews-help@spark.apache.org; run by ezmlm
Precedence: bulk
From: srowen <git@git.apache.org>
To: reviews@spark.apache.org
Reply-To: reviews@spark.apache.org
References: <git-pr-16344-spark@git.apache.org>
In-Reply-To: <git-pr-16344-spark@git.apache.org>
Subject: [GitHub] spark pull request #16344: [SPARK-18929][ML] Add Tweedie distribution in GLM
Content-Type: text/plain
Message-Id: <20161222121355.8E5D0DFBA3@git1-us-west.apache.org>
Date: Thu, 22 Dec 2016 12:13:55 +0000 (UTC)
archived-at: Thu, 22 Dec 2016 12:13:57 -0000

Github user srowen commented on a diff in the pull request:

    https://github.com/apache/spark/pull/16344#discussion_r93612965
  
    --- Diff: mllib/src/main/scala/org/apache/spark/ml/regression/GeneralizedLinearRegression.scala ---
    @@ -397,49 +436,132 @@ object GeneralizedLinearRegression extends DefaultParamsReadable[GeneralizedLine
     
         /** Trim the fitted value so that it will be in valid range. */
         def project(mu: Double): Double = mu
    +
       }
     
       private[regression] object Family {
     
         /**
    -     * Gets the [[Family]] object from its name.
    +     * Gets the [[Family]] object based on family and variancePower.
    +     * 1) retrieve object based on family name
    +     * 2) if family name is tweedie, retrieve object based on variancePower
          *
    -     * @param name family name: "gaussian", "binomial", "poisson" or "gamma".
    +     * @param model a GenerealizedLinearRegressionBase object
          */
    -    def fromName(name: String): Family = {
    -      name match {
    -        case Gaussian.name => Gaussian
    -        case Binomial.name => Binomial
    -        case Poisson.name => Poisson
    -        case Gamma.name => Gamma
    +    def fromModel(model: GeneralizedLinearRegressionBase): Family = {
    +      model.getFamily match {
    +        case "gaussian" => Gaussian
    +        case "binomial" => Binomial
    +        case "poisson" => Poisson
    +        case "gamma" => Gamma
    +        case "tweedie" =>
    +          model.getVariancePower match {
    +            case 0.0 => Gaussian
    +            case 1.0 => Poisson
    +            case 2.0 => Gamma
    +            case default => new TweedieFamily(default)
    +          }
           }
         }
       }
     
       /**
    -   * Gaussian exponential family distribution.
    -   * The default link for the Gaussian family is the identity link.
    -   */
    -  private[regression] object Gaussian extends Family("gaussian") {
    +    * Tweedie exponential family distribution.
    +    * This includes the special cases of Gaussian, Poisson and Gamma.
    +    */
    +  private[regression] class TweedieFamily(private val variancePower: Double)
    +    extends Family{
    +
    +    val name: String = variancePower match {
    +      case 0.0 => "gaussian"
    +      case 1.0 => "poisson"
    +      case 2.0 => "gamma"
    +      case default => "tweedie"
    +    }
    +    /*
    +      The canonical link is 1 - variancePower. Except for the special cases of Gaussian,
    +      Poisson and Gamma, the canonical link is rarely used. Set Log as the default link.
    +    */
    +    val defaultLink: Link = variancePower match {
    +      case 0.0 => Identity
    +      case 1.0 => Log
    +      case 2.0 => Inverse
    +      case _ => Log
    +    }
     
    -    val defaultLink: Link = Identity
    +    override def initialize(y: Double, weight: Double): Double = {
    +      if (variancePower >= 1.0 && variancePower < 2.0) {
    +        require(y >= 0.0, s"The response variable of the specified $name distribution " +
    +          s"should be non-negative, but got $y")
    +      } else if (variancePower >= 2.0) {
    +        require(y > 0.0, s"The response variable of the specified $name distribution " +
    +          s"should be non-negative, but got $y")
    +      }
    +      if (y == 0) delta else y
    +    }
     
    -    override def initialize(y: Double, weight: Double): Double = y
    +    override def variance(mu: Double): Double = {
    +      variancePower match {
    +        case 0.0 => 1.0
    +        case 1.0 => mu
    +        case 2.0 => mu * mu
    +        case default => math.pow(mu, default)
    +      }
    +    }
     
    -    override def variance(mu: Double): Double = 1.0
    +    private def yp(y: Double, mu: Double, p: Double): Double = {
    +      if (p == 0) {
    +        math.log(y / mu)
    +      } else {
    +        (math.pow(y, p) - math.pow(mu, p)) / p
    +      }
    +    }
     
         override def deviance(y: Double, mu: Double, weight: Double): Double = {
    -      weight * (y - mu) * (y - mu)
    +      // Force y >= delta for Poisson or compound Poisson
    +      val y1 = if (variancePower >= 1.0 && variancePower < 2.0) math.max(y, delta) else y
    +      2.0 * weight *
    +        (y * yp(y1, mu, 1.0 - variancePower) - yp(y, mu, 2.0 - variancePower))
         }
     
    -    override def aic(
    -        predictions: RDD[(Double, Double, Double)],
    -        deviance: Double,
    -        numInstances: Double,
    -        weightSum: Double): Double = {
    -      val wt = predictions.map(x => math.log(x._3)).sum()
    -      numInstances * (math.log(deviance / numInstances * 2.0 * math.Pi) + 1.0) + 2.0 - wt
    +    override def aic(predictions: RDD[(Double, Double, Double)],
    --- End diff --
    
    Likewise there's not a lot of value in pushing 4 separate implementations into one method and using if-else. Just override this method.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org