spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From etrain <...@git.apache.org>
Subject [GitHub] spark pull request: [SPARK-4789] [SPARK-4942] [SPARK-5031] [mllib]...
Date Thu, 08 Jan 2015 23:58:19 GMT
Github user etrain commented on a diff in the pull request:

    https://github.com/apache/spark/pull/3637#discussion_r22693206
  
    --- Diff: mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
---
    @@ -80,69 +50,157 @@ class LogisticRegression extends Estimator[LogisticRegressionModel]
with Logisti
     
       def setRegParam(value: Double): this.type = set(regParam, value)
       def setMaxIter(value: Int): this.type = set(maxIter, value)
    -  def setLabelCol(value: String): this.type = set(labelCol, value)
       def setThreshold(value: Double): this.type = set(threshold, value)
    -  def setFeaturesCol(value: String): this.type = set(featuresCol, value)
    -  def setScoreCol(value: String): this.type = set(scoreCol, value)
    -  def setPredictionCol(value: String): this.type = set(predictionCol, value)
     
       override def fit(dataset: SchemaRDD, paramMap: ParamMap): LogisticRegressionModel =
{
    +    // Check schema
         transformSchema(dataset.schema, paramMap, logging = true)
    -    import dataset.sqlContext._
    +
    +    // Extract columns from data.  If dataset is persisted, do not persist oldDataset.
    +    val oldDataset = extractLabeledPoints(dataset, paramMap)
         val map = this.paramMap ++ paramMap
    -    val instances = dataset.select(map(labelCol).attr, map(featuresCol).attr)
    -      .map { case Row(label: Double, features: Vector) =>
    -        LabeledPoint(label, features)
    -      }.persist(StorageLevel.MEMORY_AND_DISK)
    +    val handlePersistence = dataset.getStorageLevel == StorageLevel.NONE
    +    if (handlePersistence) {
    +      oldDataset.persist(StorageLevel.MEMORY_AND_DISK)
    +    }
    +
    +    // Train model
         val lr = new LogisticRegressionWithLBFGS
         lr.optimizer
           .setRegParam(map(regParam))
           .setNumIterations(map(maxIter))
    -    val lrm = new LogisticRegressionModel(this, map, lr.run(instances).weights)
    -    instances.unpersist()
    +    val oldModel = lr.run(oldDataset)
    +    val lrm = new LogisticRegressionModel(this, map, oldModel.weights, oldModel.intercept)
    +
    +    if (handlePersistence) {
    +      oldDataset.unpersist()
    +    }
    +
         // copy model params
         Params.inheritValues(map, this, lrm)
         lrm
       }
     
    -  private[ml] override def transformSchema(schema: StructType, paramMap: ParamMap): StructType
= {
    -    validateAndTransformSchema(schema, paramMap, fitting = true)
    -  }
    +  override protected def featuresDataType: DataType = new VectorUDT
    --- End diff --
    
    What is this for?


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message