airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject incubator-airflow git commit: [AIRFLOW-1763] Fix S3TaskHandler unit tests
Date Tue, 31 Oct 2017 18:32:19 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/master c5776375f -> 44710d7e9


[AIRFLOW-1763] Fix S3TaskHandler unit tests

Fix breaking S3TaskHandler unit tests, and create
a package so that
tests are identified by CI.

Closes #2732 from andyxhadji/AIRFLOW-1763


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/44710d7e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/44710d7e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/44710d7e

Branch: refs/heads/master
Commit: 44710d7e9ef8b8955ff44da2f55cfb0e0774629e
Parents: c577637
Author: Andy Hadjigeorgiou <ahh2131@columbia.edu>
Authored: Tue Oct 31 19:32:13 2017 +0100
Committer: Bolke de Bruin <bolke@xs4all.nl>
Committed: Tue Oct 31 19:32:13 2017 +0100

----------------------------------------------------------------------
 tests/utils/log/test_s3_task_handler.py | 54 ++++++++++++++++------------
 1 file changed, 32 insertions(+), 22 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/44710d7e/tests/utils/log/test_s3_task_handler.py
----------------------------------------------------------------------
diff --git a/tests/utils/log/test_s3_task_handler.py b/tests/utils/log/test_s3_task_handler.py
index 0438630..da879b6 100644
--- a/tests/utils/log/test_s3_task_handler.py
+++ b/tests/utils/log/test_s3_task_handler.py
@@ -18,76 +18,84 @@ import unittest
 from airflow.utils.log.s3_task_handler import S3TaskHandler
 
 
-@unittest.skip("Non functional S3 tests")
 class TestS3TaskHandler(unittest.TestCase):
 
     def setUp(self):
         super(TestS3TaskHandler, self).setUp()
         self.remote_log_location = 'remote/log/location'
+        self.local_log_location = 'local/log/location'
+        self.s3_log_location = 's3/log/location'
+        self.filename_template = ''
         self.hook_patcher = mock.patch("airflow.hooks.S3_hook.S3Hook")
         self.hook_mock = self.hook_patcher.start()
         self.hook_inst_mock = self.hook_mock.return_value
         self.hook_key_mock = self.hook_inst_mock.get_key.return_value
         self.hook_key_mock.get_contents_as_string.return_value.decode.\
             return_value = 'content'
+        self.s3_task_handler = S3TaskHandler(
+            self.local_log_location,
+            self.s3_log_location,
+            self.filename_template
+        )
 
     def tearDown(self):
         self.hook_patcher.stop()
         super(TestS3TaskHandler, self).tearDown()
 
     def test_init(self):
-        S3TaskHandler()
+        self.s3_task_handler.hook()
         self.hook_mock.assert_called_once_with('')
 
     def test_init_raises(self):
         self.hook_mock.side_effect = Exception('Failed to connect')
-        handler = S3TaskHandler()
+        handler = self.s3_task_handler
         with mock.patch.object(handler.log, 'error') as mock_error:
             # Initialize the hook
-            handler.hook()
+            handler.hook
             mock_error.assert_called_once_with(
-                'Could not create an S3Hook with connection id "". Please make '
-                'sure that airflow[s3] is installed and the S3 connection exists.'
+                'Could not create an S3Hook with connection id "%s". Please make '
+                'sure that airflow[s3] is installed and the S3 connection exists.',
+                ''
             )
 
     def test_log_exists(self):
-        self.assertTrue(S3TaskHandler().log_exists(self.remote_log_location))
+        self.assertTrue(self.s3_task_handler.s3_log_exists(self.remote_log_location))
 
     def test_log_exists_none(self):
         self.hook_inst_mock.get_key.return_value = None
-        self.assertFalse(S3TaskHandler().log_exists(self.remote_log_location))
+        self.assertFalse(self.s3_task_handler.s3_log_exists(self.remote_log_location))
 
     def test_log_exists_raises(self):
         self.hook_inst_mock.get_key.side_effect = Exception('error')
-        self.assertFalse(S3TaskHandler().log_exists(self.remote_log_location))
+        self.assertFalse(self.s3_task_handler.s3_log_exists(self.remote_log_location))
 
     def test_log_exists_false(self):
-        self.hook_inst_mock.check_for_key.return_value = False
-        self.assertFalse(S3TaskHandler().log_exists(self.remote_log_location))
+        self.hook_inst_mock.get_key.return_value = None
+        self.assertFalse(self.s3_task_handler.s3_log_exists(self.remote_log_location))
 
     def test_log_exists_no_hook(self):
         self.hook_mock.side_effect = Exception('Failed to connect')
-        self.assertFalse(S3TaskHandler().log_exists(self.remote_log_location))
+        self.assertFalse(self.s3_task_handler.s3_log_exists(self.remote_log_location))
 
     def test_read(self):
         self.assertEqual(
-            S3TaskHandler().read(self.remote_log_location),
+            self.s3_task_handler.s3_read(self.remote_log_location),
             'content'
         )
 
     def test_read_key_empty(self):
         self.hook_inst_mock.get_key.return_value = None
-        self.assertEqual(S3TaskHandler().read(self.remote_log_location), '')
+        self.assertEqual(self.s3_task_handler.s3_read(self.remote_log_location), None)
 
     def test_read_raises(self):
         self.hook_inst_mock.get_key.side_effect = Exception('error')
-        self.assertEqual(S3TaskHandler().read(self.remote_log_location), '')
+        self.assertEqual(self.s3_task_handler.s3_read(self.remote_log_location), None)
 
     def test_read_raises_return_error(self):
         self.hook_inst_mock.get_key.side_effect = Exception('error')
-        handler = S3TaskHandler()
+        handler = self.s3_task_handler
         with mock.patch.object(handler.log, 'error') as mock_error:
-            result = handler.s3_log_read(
+            result = handler.s3_read(
                 self.remote_log_location,
                 return_error=True
             )
@@ -96,7 +104,7 @@ class TestS3TaskHandler(unittest.TestCase):
             mock_error.assert_called_once_with(msg)
 
     def test_write(self):
-        S3TaskHandler().write('text', self.remote_log_location)
+        self.s3_task_handler.s3_write('text', self.remote_log_location)
         self.hook_inst_mock.load_string.assert_called_once_with(
             'content\ntext',
             key=self.remote_log_location,
@@ -107,8 +115,10 @@ class TestS3TaskHandler(unittest.TestCase):
     def test_write_raises(self):
         self.hook_inst_mock.read_key.return_value = ''
         self.hook_inst_mock.load_string.side_effect = Exception('error')
-        handler = S3TaskHandler()
+        handler = self.s3_task_handler
         with mock.patch.object(handler.log, 'error') as mock_error:
-            handler.write('text', self.remote_log_location)
-            msg = 'Could not write logs to %s' % self.remote_log_location
-            mock_error.assert_called_once_with(msg)
+            handler.s3_write('text', self.remote_log_location)
+            mock_error.assert_called_once_with(
+                'Could not write logs to %s',
+                'remote/log/location'
+            )


Mime
View raw message