flink-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From trohrm...@apache.org
Subject [2/2] flink git commit: [ml] Makes StandardScalers state package private and reduce redundant code. Adjusts flink-ml readme.
Date Tue, 09 Jun 2015 08:37:53 GMT
[ml] Makes StandardScalers state package private and reduce redundant code. Adjusts flink-ml
readme.


Project: http://git-wip-us.apache.org/repos/asf/flink/repo
Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/80cf2c54
Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/80cf2c54
Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/80cf2c54

Branch: refs/heads/release-0.9
Commit: 80cf2c544f887532d0c5946027a2adaa481a16d0
Parents: c83ee8a
Author: Till Rohrmann <trohrmann@apache.org>
Authored: Tue Jun 9 10:23:09 2015 +0200
Committer: Till Rohrmann <trohrmann@apache.org>
Committed: Tue Jun 9 10:35:57 2015 +0200

----------------------------------------------------------------------
 flink-staging/flink-ml/README.md                |  5 +--
 .../flink/ml/preprocessing/StandardScaler.scala | 44 +++++++++++++-------
 2 files changed, 32 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/flink/blob/80cf2c54/flink-staging/flink-ml/README.md
----------------------------------------------------------------------
diff --git a/flink-staging/flink-ml/README.md b/flink-staging/flink-ml/README.md
index 5721e8f..5cabd7c 100644
--- a/flink-staging/flink-ml/README.md
+++ b/flink-staging/flink-ml/README.md
@@ -7,10 +7,9 @@ Theses implementations allow to scale to data sizes which vastly exceed the
memo
 Flink-ML currently comprises the following algorithms:
 
 * Classification
+** Soft-margin SVM
 * Regression
-** Logistic regression
-* Clustering
-** k-Means
+** Multiple linear regression
 * Recommendation
 ** Alternating least squares (ALS)
 

http://git-wip-us.apache.org/repos/asf/flink/blob/80cf2c54/flink-staging/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/StandardScaler.scala
----------------------------------------------------------------------
diff --git a/flink-staging/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/StandardScaler.scala
b/flink-staging/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/StandardScaler.scala
index 3b9c8d2..bf09b20 100644
--- a/flink-staging/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/StandardScaler.scala
+++ b/flink-staging/flink-ml/src/main/scala/org/apache/flink/ml/preprocessing/StandardScaler.scala
@@ -21,10 +21,8 @@ package org.apache.flink.ml.preprocessing
 import breeze.linalg
 import breeze.numerics.sqrt
 import breeze.numerics.sqrt._
-import org.apache.flink.api.common.functions._
 import org.apache.flink.api.common.typeinfo.TypeInformation
 import org.apache.flink.api.scala._
-import org.apache.flink.configuration.Configuration
 import org.apache.flink.ml._
 import org.apache.flink.ml.common.{LabeledVector, Parameter, ParameterMap}
 import org.apache.flink.ml.math.Breeze._
@@ -62,7 +60,9 @@ import scala.reflect.ClassTag
   */
 class StandardScaler extends Transformer[StandardScaler] {
 
-  var metricsOption: Option[DataSet[(linalg.Vector[Double], linalg.Vector[Double])]] = None
+  private[preprocessing] var metricsOption: Option[
+      DataSet[(linalg.Vector[Double], linalg.Vector[Double])]
+    ] = None
 
   /** Sets the target mean of the transformed data
     *
@@ -213,12 +213,7 @@ object StandardScaler {
             input.mapWithBcVariable(metrics){
               (vector, metrics) => {
                 val (broadcastMean, broadcastStd) = metrics
-                var myVector = vector.asBreeze
-
-                myVector -= broadcastMean
-                myVector :/= broadcastStd
-                myVector = (myVector :* std) + mean
-                myVector.fromBreeze
+                scaleVector(vector, broadcastMean, broadcastStd, mean, std)
               }
             }
           }
@@ -245,12 +240,8 @@ object StandardScaler {
               (labeledVector, metrics) => {
                 val (broadcastMean, broadcastStd) = metrics
                 val LabeledVector(label, vector) = labeledVector
-                var breezeVector = vector.asBreeze
 
-                breezeVector -= broadcastMean
-                breezeVector :/= broadcastStd
-                breezeVector = (breezeVector :* std) + mean
-                LabeledVector(label, breezeVector.fromBreeze)
+                LabeledVector(label, scaleVector(vector, broadcastMean, broadcastStd, mean,
std))
               }
             }
           }
@@ -262,4 +253,29 @@ object StandardScaler {
       }
     }
   }
+
+  /** Scales the given vector such that it has the given mean and std
+    *
+    * @param vector Vector to be scaled
+    * @param dataMean Mean of the training data
+    * @param dataStd Standard deviation of the training data
+    * @param mean Mean of the scaled data
+    * @param std Standard deviation of the scaled data
+    * @tparam T Type of [[Vector]]
+    * @return Scaled vector
+    */
+  private def scaleVector[T <: Vector: BreezeVectorConverter](
+      vector: T,
+      dataMean: linalg.Vector[Double],
+      dataStd: linalg.Vector[Double],
+      mean: Double,
+      std: Double)
+    : T = {
+    var myVector = vector.asBreeze
+
+    myVector -= dataMean
+    myVector :/= dataStd
+    myVector = (myVector :* std) + mean
+    myVector.fromBreeze
+  }
 }


Mime
View raw message