airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject incubator-airflow git commit: [AIRFLOW-1326][[AIRFLOW-1326][AIRFLOW-1184] Don't split argument array -- it's already an array.[
Date Wed, 12 Jul 2017 09:27:46 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/master 71594751c -> f5ffb9deb


[AIRFLOW-1326][[AIRFLOW-1326][AIRFLOW-1184] Don't split argument array -- it's already an
array.[

Closes #2382 from ashb/spark-submit-operator-
preserve-spaces


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/f5ffb9de
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/f5ffb9de
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/f5ffb9de

Branch: refs/heads/master
Commit: f5ffb9deb51e896cdda8992c8b28102f7d650662
Parents: 7159475
Author: Ash Berlin-Taylor <ash_github@firemirror.com>
Authored: Wed Jul 12 11:27:39 2017 +0200
Committer: Bolke de Bruin <bolke@xs4all.nl>
Committed: Wed Jul 12 11:27:39 2017 +0200

----------------------------------------------------------------------
 airflow/contrib/hooks/spark_submit_hook.py      |  8 ++---
 tests/contrib/hooks/test_spark_submit_hook.py   |  6 ++--
 .../operators/test_spark_submit_operator.py     | 31 +++++++++++---------
 3 files changed, 23 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f5ffb9de/airflow/contrib/hooks/spark_submit_hook.py
----------------------------------------------------------------------
diff --git a/airflow/contrib/hooks/spark_submit_hook.py b/airflow/contrib/hooks/spark_submit_hook.py
index f3d6e34..88d547b 100644
--- a/airflow/contrib/hooks/spark_submit_hook.py
+++ b/airflow/contrib/hooks/spark_submit_hook.py
@@ -194,12 +194,8 @@ class SparkSubmitHook(BaseHook):
 
         # Append any application arguments
         if self._application_args:
-            for arg in self._application_args:
-                if len(arg.split()) > 1:
-                    for splitted_option in arg.split():
-                        connection_cmd += [splitted_option]
-                else:
-                    connection_cmd += [arg]
+            connection_cmd += self._application_args
+
         logging.debug("Spark-Submit cmd: {}".format(connection_cmd))
 
         return connection_cmd

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f5ffb9de/tests/contrib/hooks/test_spark_submit_hook.py
----------------------------------------------------------------------
diff --git a/tests/contrib/hooks/test_spark_submit_hook.py b/tests/contrib/hooks/test_spark_submit_hook.py
index ef0857b..98e959b 100644
--- a/tests/contrib/hooks/test_spark_submit_hook.py
+++ b/tests/contrib/hooks/test_spark_submit_hook.py
@@ -45,8 +45,9 @@ class TestSparkSubmitHook(unittest.TestCase):
         'driver_memory': '3g',
         'java_class': 'com.foo.bar.AppMain',
         'application_args': [
-            '-f foo',
-            '--bar bar',
+            '-f', 'foo',
+            '--bar', 'bar',
+            '--with-spaces', 'args should keep embdedded spaces',
             'baz'
         ]
     }
@@ -129,6 +130,7 @@ class TestSparkSubmitHook(unittest.TestCase):
             'test_application.py',
             '-f', 'foo',
             '--bar', 'bar',
+            '--with-spaces', 'args should keep embdedded spaces',
             'baz'
         ]
         self.assertEquals(expected_build_cmd, cmd)

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/f5ffb9de/tests/contrib/operators/test_spark_submit_operator.py
----------------------------------------------------------------------
diff --git a/tests/contrib/operators/test_spark_submit_operator.py b/tests/contrib/operators/test_spark_submit_operator.py
index 09c5a93..1d41941 100644
--- a/tests/contrib/operators/test_spark_submit_operator.py
+++ b/tests/contrib/operators/test_spark_submit_operator.py
@@ -46,10 +46,11 @@ class TestSparkSubmitOperator(unittest.TestCase):
         'driver_memory': '3g',
         'java_class': 'com.foo.bar.AppMain',
         'application_args': [
-            '-f foo',
-            '--bar bar',
-            '--start {{ macros.ds_add(ds, -1)}}',
-            '--end {{ ds }}'
+            '-f', 'foo',
+            '--bar', 'bar',
+            '--start', '{{ macros.ds_add(ds, -1)}}',
+            '--end', '{{ ds }}',
+            '--with-spaces', 'args should keep embdedded spaces',
         ]
     }
 
@@ -95,10 +96,11 @@ class TestSparkSubmitOperator(unittest.TestCase):
             'driver_memory': '3g',
             'java_class': 'com.foo.bar.AppMain',
             'application_args': [
-                '-f foo',
-                '--bar bar',
-                '--start {{ macros.ds_add(ds, -1)}}',
-                '--end {{ ds }}'
+                '-f', 'foo',
+                '--bar', 'bar',
+                '--start', '{{ macros.ds_add(ds, -1)}}',
+                '--end', '{{ ds }}',
+                '--with-spaces', 'args should keep embdedded spaces',
             ]
 
         }
@@ -130,14 +132,15 @@ class TestSparkSubmitOperator(unittest.TestCase):
         ti.render_templates()
 
         # Then
-        expected_application_args = [u'-f foo',
-                                     u'--bar bar',
-                                     u'--start %s' % (DEFAULT_DATE - datetime.timedelta(days=1)).strftime("%Y-%m-%d"),
-                                     u'--end %s' % DEFAULT_DATE.strftime("%Y-%m-%d")]
+        expected_application_args = [u'-f', 'foo',
+                                     u'--bar', 'bar',
+                                     u'--start', (DEFAULT_DATE - datetime.timedelta(days=1)).strftime("%Y-%m-%d"),
+                                     u'--end', DEFAULT_DATE.strftime("%Y-%m-%d"),
+                                     u'--with-spaces', u'args should keep embdedded spaces',
+                                     ]
         expected_name = "spark_submit_job"
-        self.assertListEqual(sorted(expected_application_args), sorted(getattr(operator,
'_application_args')))
+        self.assertListEqual(expected_application_args, getattr(operator, '_application_args'))
         self.assertEqual(expected_name, getattr(operator, '_name'))
 
-
 if __name__ == '__main__':
     unittest.main()


Mime
View raw message