spark-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From ma...@apache.org
Subject git commit: SPARK-1917: fix PySpark import of scipy.special functions
Date Sat, 31 May 2014 21:59:12 GMT
Repository: spark
Updated Branches:
  refs/heads/master d8c005d53 -> 5e98967b6


SPARK-1917: fix PySpark import of scipy.special functions

https://issues.apache.org/jira/browse/SPARK-1917

Author: Uri Laserson <laserson@cloudera.com>

Closes #866 from laserson/SPARK-1917 and squashes the following commits:

d947e8c [Uri Laserson] Added test for scipy.special importing
1798bbd [Uri Laserson] SPARK-1917: fix PySpark import of scipy.special


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/5e98967b
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/5e98967b
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/5e98967b

Branch: refs/heads/master
Commit: 5e98967b612ccf026cb1cc5ff3ac8bf72d7e836e
Parents: d8c005d
Author: Uri Laserson <laserson@cloudera.com>
Authored: Sat May 31 14:59:09 2014 -0700
Committer: Matei Zaharia <matei@databricks.com>
Committed: Sat May 31 14:59:09 2014 -0700

----------------------------------------------------------------------
 python/pyspark/cloudpickle.py |  2 +-
 python/pyspark/tests.py       | 24 ++++++++++++++++++++++++
 2 files changed, 25 insertions(+), 1 deletion(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/spark/blob/5e98967b/python/pyspark/cloudpickle.py
----------------------------------------------------------------------
diff --git a/python/pyspark/cloudpickle.py b/python/pyspark/cloudpickle.py
index 6a7c23a..eb5dbb8 100644
--- a/python/pyspark/cloudpickle.py
+++ b/python/pyspark/cloudpickle.py
@@ -933,7 +933,7 @@ def _change_cell_value(cell, newval):
 Note: These can never be renamed due to client compatibility issues"""
 
 def _getobject(modname, attribute):
-    mod = __import__(modname)
+    mod = __import__(modname, fromlist=[attribute])
     return mod.__dict__[attribute]
 
 def _generateImage(size, mode, str_rep):

http://git-wip-us.apache.org/repos/asf/spark/blob/5e98967b/python/pyspark/tests.py
----------------------------------------------------------------------
diff --git a/python/pyspark/tests.py b/python/pyspark/tests.py
index 64f2eeb..ed90915 100644
--- a/python/pyspark/tests.py
+++ b/python/pyspark/tests.py
@@ -35,6 +35,14 @@ from pyspark.context import SparkContext
 from pyspark.files import SparkFiles
 from pyspark.serializers import read_int
 
+_have_scipy = False
+try:
+    import scipy.sparse
+    _have_scipy = True
+except:
+    # No SciPy, but that's okay, we'll skip those tests
+    pass
+
 
 SPARK_HOME = os.environ["SPARK_HOME"]
 
@@ -359,5 +367,21 @@ class TestSparkSubmit(unittest.TestCase):
         self.assertIn("[2, 4, 6]", out)
 
 
+@unittest.skipIf(not _have_scipy, "SciPy not installed")
+class SciPyTests(PySparkTestCase):
+    """General PySpark tests that depend on scipy """
+
+    def test_serialize(self):
+        from scipy.special import gammaln
+        x = range(1, 5)
+        expected = map(gammaln, x)
+        observed = self.sc.parallelize(x).map(gammaln).collect()
+        self.assertEqual(expected, observed)
+
+
 if __name__ == "__main__":
+    if not _have_scipy:
+        print "NOTE: Skipping SciPy tests as it does not seem to be installed"
     unittest.main()
+    if not _have_scipy:
+        print "NOTE: SciPy tests were skipped as it does not seem to be installed"


Mime
View raw message