spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mln...@apache.org
Subject spark git commit: [MINOR][ML][PYSPARK] ALS example cleanup
Date Sat, 07 May 2016 08:58:08 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-2.0 49e666138 -> d0302a28c


[MINOR][ML][PYSPARK] ALS example cleanup

Cleans up ALS examples by removing unnecessary casts to double for `rating` and `prediction`
columns, since `RegressionEvaluator` now supports `Double` & `Float` input types.

## How was this patch tested?

Manual compile and run with `run-example ml.ALSExample` and `spark-submit examples/src/main/python/ml/als_example.py`.

Author: Nick Pentreath <nickp@za.ibm.com>

Closes #12892 from MLnick/als-examples-cleanup.

(cherry picked from commit b0cafdb6ccff9add89dc31c45adf87c8fa906aac)
Signed-off-by: Nick Pentreath <nickp@za.ibm.com>


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d0302a28
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d0302a28
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d0302a28

Branch: refs/heads/branch-2.0
Commit: d0302a28c1660c63acdea9a30a438f0172a19c56
Parents: 49e6661
Author: Nick Pentreath <nickp@za.ibm.com>
Authored: Sat May 7 10:57:40 2016 +0200
Committer: Nick Pentreath <nickp@za.ibm.com>
Committed: Sat May 7 10:57:59 2016 +0200

----------------------------------------------------------------------
 .../java/org/apache/spark/examples/ml/JavaALSExample.java   | 6 +-----
 examples/src/main/python/ml/als_example.py                  | 9 +++------
 .../scala/org/apache/spark/examples/ml/ALSExample.scala     | 6 ------
 3 files changed, 4 insertions(+), 17 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/d0302a28/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java
----------------------------------------------------------------------
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java
index 4b13ba6..7f568f4 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaALSExample.java
@@ -29,7 +29,6 @@ import org.apache.spark.api.java.function.Function;
 import org.apache.spark.ml.evaluation.RegressionEvaluator;
 import org.apache.spark.ml.recommendation.ALS;
 import org.apache.spark.ml.recommendation.ALSModel;
-import org.apache.spark.sql.types.DataTypes;
 // $example off$
 
 public class JavaALSExample {
@@ -109,10 +108,7 @@ public class JavaALSExample {
     ALSModel model = als.fit(training);
 
     // Evaluate the model by computing the RMSE on the test data
-    Dataset<Row> rawPredictions = model.transform(test);
-    Dataset<Row> predictions = rawPredictions
-      .withColumn("rating", rawPredictions.col("rating").cast(DataTypes.DoubleType))
-      .withColumn("prediction", rawPredictions.col("prediction").cast(DataTypes.DoubleType));
+    Dataset<Row> predictions = model.transform(test);
 
     RegressionEvaluator evaluator = new RegressionEvaluator()
       .setMetricName("rmse")

http://git-wip-us.apache.org/repos/asf/spark/blob/d0302a28/examples/src/main/python/ml/als_example.py
----------------------------------------------------------------------
diff --git a/examples/src/main/python/ml/als_example.py b/examples/src/main/python/ml/als_example.py
index ff0829b..1a979ff 100644
--- a/examples/src/main/python/ml/als_example.py
+++ b/examples/src/main/python/ml/als_example.py
@@ -48,12 +48,9 @@ if __name__ == "__main__":
     model = als.fit(training)
 
     # Evaluate the model by computing the RMSE on the test data
-    rawPredictions = model.transform(test)
-    predictions = rawPredictions\
-        .withColumn("rating", rawPredictions.rating.cast("double"))\
-        .withColumn("prediction", rawPredictions.prediction.cast("double"))
-    evaluator =\
-        RegressionEvaluator(metricName="rmse", labelCol="rating", predictionCol="prediction")
+    predictions = model.transform(test)
+    evaluator = RegressionEvaluator(metricName="rmse", labelCol="rating",
+                                    predictionCol="prediction")
     rmse = evaluator.evaluate(predictions)
     print("Root-mean-square error = " + str(rmse))
     # $example off$

http://git-wip-us.apache.org/repos/asf/spark/blob/d0302a28/examples/src/main/scala/org/apache/spark/examples/ml/ALSExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/ALSExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/ALSExample.scala
index 7c1cfe2..6b151a6 100644
--- a/examples/src/main/scala/org/apache/spark/examples/ml/ALSExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/ml/ALSExample.scala
@@ -23,10 +23,6 @@ import org.apache.spark.ml.evaluation.RegressionEvaluator
 import org.apache.spark.ml.recommendation.ALS
 // $example off$
 import org.apache.spark.sql.SparkSession
-// $example on$
-import org.apache.spark.sql.functions._
-import org.apache.spark.sql.types.DoubleType
-// $example off$
 
 object ALSExample {
 
@@ -65,8 +61,6 @@ object ALSExample {
 
     // Evaluate the model by computing the RMSE on the test data
     val predictions = model.transform(test)
-      .withColumn("rating", col("rating").cast(DoubleType))
-      .withColumn("prediction", col("prediction").cast(DoubleType))
 
     val evaluator = new RegressionEvaluator()
       .setMetricName("rmse")


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message