spark-reviews mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From viirya <...@git.apache.org>
Subject [GitHub] spark pull request #21467: [SPARK-23754][PYTHON][FOLLOWUP] Move UDF stop ite...
Date Wed, 06 Jun 2018 06:43:47 GMT
Github user viirya commented on a diff in the pull request:

    https://github.com/apache/spark/pull/21467#discussion_r193306421
  
    --- Diff: python/pyspark/tests.py ---
    @@ -1291,27 +1291,31 @@ def test_pipe_unicode(self):
             result = rdd.pipe('cat').collect()
             self.assertEqual(data, result)
     
    -    def test_stopiteration_in_client_code(self):
    +    def test_stopiteration_in_user_code(self):
     
             def stopit(*x):
                 raise StopIteration()
     
             seq_rdd = self.sc.parallelize(range(10))
             keyed_rdd = self.sc.parallelize((x % 2, x) for x in range(10))
    +        msg = "Caught StopIteration thrown from user's code; failing the task"
     
    -        self.assertRaises(Py4JJavaError, seq_rdd.map(stopit).collect)
    -        self.assertRaises(Py4JJavaError, seq_rdd.filter(stopit).collect)
    -        self.assertRaises(Py4JJavaError, seq_rdd.cartesian(seq_rdd).flatMap(stopit).collect)
    -        self.assertRaises(Py4JJavaError, seq_rdd.foreach, stopit)
    -        self.assertRaises(Py4JJavaError, keyed_rdd.reduceByKeyLocally, stopit)
    -        self.assertRaises(Py4JJavaError, seq_rdd.reduce, stopit)
    -        self.assertRaises(Py4JJavaError, seq_rdd.fold, 0, stopit)
    +        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.map(stopit).collect)
    +        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.filter(stopit).collect)
    +        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.foreach, stopit)
    +        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.reduce, stopit)
    +        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.fold, 0, stopit)
    +        self.assertRaisesRegexp(Py4JJavaError, msg, seq_rdd.foreach, stopit)
    +        self.assertRaisesRegexp(Py4JJavaError, msg,
    +                                seq_rdd.cartesian(seq_rdd).flatMap(stopit).collect)
     
             # the exception raised is non-deterministic
    --- End diff --
    
    What does this mean? The exception is non-deterministic?


---

---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org


Mime
View raw message