spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jkbrad...@apache.org
Subject spark git commit: [SPARK-11978][ML] Move dataset_example.py to examples/ml and rename to dataframe_example.py
Date Sat, 12 Dec 2015 02:02:40 GMT
Repository: spark
Updated Branches:
  refs/heads/branch-1.6 75531c77e -> c2f20469d


[SPARK-11978][ML] Move dataset_example.py to examples/ml and rename to dataframe_example.py

Since ```Dataset``` has a new meaning in Spark 1.6, we should rename it to avoid confusion.
#9873 finished the work of Scala example, here we focus on the Python one.
Move dataset_example.py to ```examples/ml``` and rename to ```dataframe_example.py```.
BTW, fix minor missing issues of #9873.
cc mengxr

Author: Yanbo Liang <ybliang8@gmail.com>

Closes #9957 from yanboliang/SPARK-11978.

(cherry picked from commit a0ff6d16ef4bcc1b6ff7282e82a9b345d8449454)
Signed-off-by: Joseph K. Bradley <joseph@databricks.com>


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c2f20469
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c2f20469
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c2f20469

Branch: refs/heads/branch-1.6
Commit: c2f20469d5b53a027b022e3c4a9bea57452c5ba6
Parents: 75531c7
Author: Yanbo Liang <ybliang8@gmail.com>
Authored: Fri Dec 11 18:02:24 2015 -0800
Committer: Joseph K. Bradley <joseph@databricks.com>
Committed: Fri Dec 11 18:02:37 2015 -0800

----------------------------------------------------------------------
 .../src/main/python/ml/dataframe_example.py     | 75 ++++++++++++++++++++
 .../src/main/python/mllib/dataset_example.py    | 63 ----------------
 .../spark/examples/ml/DataFrameExample.scala    |  8 +--
 3 files changed, 79 insertions(+), 67 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/c2f20469/examples/src/main/python/ml/dataframe_example.py
----------------------------------------------------------------------
diff --git a/examples/src/main/python/ml/dataframe_example.py b/examples/src/main/python/ml/dataframe_example.py
new file mode 100644
index 0000000..d2644ca
--- /dev/null
+++ b/examples/src/main/python/ml/dataframe_example.py
@@ -0,0 +1,75 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+"""
+An example of how to use DataFrame for ML. Run with::
+    bin/spark-submit examples/src/main/python/ml/dataframe_example.py <input>
+"""
+from __future__ import print_function
+
+import os
+import sys
+import tempfile
+import shutil
+
+from pyspark import SparkContext
+from pyspark.sql import SQLContext
+from pyspark.mllib.stat import Statistics
+
+if __name__ == "__main__":
+    if len(sys.argv) > 2:
+        print("Usage: dataframe_example.py <libsvm file>", file=sys.stderr)
+        exit(-1)
+    sc = SparkContext(appName="DataFrameExample")
+    sqlContext = SQLContext(sc)
+    if len(sys.argv) == 2:
+        input = sys.argv[1]
+    else:
+        input = "data/mllib/sample_libsvm_data.txt"
+
+    # Load input data
+    print("Loading LIBSVM file with UDT from " + input + ".")
+    df = sqlContext.read.format("libsvm").load(input).cache()
+    print("Schema from LIBSVM:")
+    df.printSchema()
+    print("Loaded training data as a DataFrame with " +
+          str(df.count()) + " records.")
+
+    # Show statistical summary of labels.
+    labelSummary = df.describe("label")
+    labelSummary.show()
+
+    # Convert features column to an RDD of vectors.
+    features = df.select("features").map(lambda r: r.features)
+    summary = Statistics.colStats(features)
+    print("Selected features column with average values:\n" +
+          str(summary.mean()))
+
+    # Save the records in a parquet file.
+    tempdir = tempfile.NamedTemporaryFile(delete=False).name
+    os.unlink(tempdir)
+    print("Saving to " + tempdir + " as Parquet file.")
+    df.write.parquet(tempdir)
+
+    # Load the records back.
+    print("Loading Parquet file with UDT from " + tempdir)
+    newDF = sqlContext.read.parquet(tempdir)
+    print("Schema from Parquet:")
+    newDF.printSchema()
+    shutil.rmtree(tempdir)
+
+    sc.stop()

http://git-wip-us.apache.org/repos/asf/spark/blob/c2f20469/examples/src/main/python/mllib/dataset_example.py
----------------------------------------------------------------------
diff --git a/examples/src/main/python/mllib/dataset_example.py b/examples/src/main/python/mllib/dataset_example.py
deleted file mode 100644
index e23ecc0..0000000
--- a/examples/src/main/python/mllib/dataset_example.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-"""
-An example of how to use DataFrame as a dataset for ML. Run with::
-    bin/spark-submit examples/src/main/python/mllib/dataset_example.py
-"""
-from __future__ import print_function
-
-import os
-import sys
-import tempfile
-import shutil
-
-from pyspark import SparkContext
-from pyspark.sql import SQLContext
-from pyspark.mllib.util import MLUtils
-from pyspark.mllib.stat import Statistics
-
-
-def summarize(dataset):
-    print("schema: %s" % dataset.schema().json())
-    labels = dataset.map(lambda r: r.label)
-    print("label average: %f" % labels.mean())
-    features = dataset.map(lambda r: r.features)
-    summary = Statistics.colStats(features)
-    print("features average: %r" % summary.mean())
-
-if __name__ == "__main__":
-    if len(sys.argv) > 2:
-        print("Usage: dataset_example.py <libsvm file>", file=sys.stderr)
-        exit(-1)
-    sc = SparkContext(appName="DatasetExample")
-    sqlContext = SQLContext(sc)
-    if len(sys.argv) == 2:
-        input = sys.argv[1]
-    else:
-        input = "data/mllib/sample_libsvm_data.txt"
-    points = MLUtils.loadLibSVMFile(sc, input)
-    dataset0 = sqlContext.inferSchema(points).setName("dataset0").cache()
-    summarize(dataset0)
-    tempdir = tempfile.NamedTemporaryFile(delete=False).name
-    os.unlink(tempdir)
-    print("Save dataset as a Parquet file to %s." % tempdir)
-    dataset0.saveAsParquetFile(tempdir)
-    print("Load it back and summarize it again.")
-    dataset1 = sqlContext.parquetFile(tempdir).setName("dataset1").cache()
-    summarize(dataset1)
-    shutil.rmtree(tempdir)

http://git-wip-us.apache.org/repos/asf/spark/blob/c2f20469/examples/src/main/scala/org/apache/spark/examples/ml/DataFrameExample.scala
----------------------------------------------------------------------
diff --git a/examples/src/main/scala/org/apache/spark/examples/ml/DataFrameExample.scala b/examples/src/main/scala/org/apache/spark/examples/ml/DataFrameExample.scala
index 424f001..0a477ab 100644
--- a/examples/src/main/scala/org/apache/spark/examples/ml/DataFrameExample.scala
+++ b/examples/src/main/scala/org/apache/spark/examples/ml/DataFrameExample.scala
@@ -44,10 +44,10 @@ object DataFrameExample {
   def main(args: Array[String]) {
     val defaultParams = Params()
 
-    val parser = new OptionParser[Params]("DatasetExample") {
-      head("Dataset: an example app using DataFrame as a Dataset for ML.")
+    val parser = new OptionParser[Params]("DataFrameExample") {
+      head("DataFrameExample: an example app using DataFrame for ML.")
       opt[String]("input")
-        .text(s"input path to dataset")
+        .text(s"input path to dataframe")
         .action((x, c) => c.copy(input = x))
       checkConfig { params =>
         success
@@ -88,7 +88,7 @@ object DataFrameExample {
     // Save the records in a parquet file.
     val tmpDir = Files.createTempDir()
     tmpDir.deleteOnExit()
-    val outputDir = new File(tmpDir, "dataset").toString
+    val outputDir = new File(tmpDir, "dataframe").toString
     println(s"Saving to $outputDir as Parquet file.")
     df.write.parquet(outputDir)
 


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org


Mime
View raw message