airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bo...@apache.org
Subject incubator-airflow git commit: [AIRFLOW-1897][AIRFLOW-1873] Task Logs for running instance not visible in WebUI
Date Sat, 09 Dec 2017 21:06:12 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/v1-9-test bc766e79c -> a3baf9e7f


[AIRFLOW-1897][AIRFLOW-1873] Task Logs for running instance not visible in WebUI

Due to the change in AIRFLOW-1873 we inadvertently
changed the behaviour
such that task logs for a try wouldn't show up in
the UI until after the
task run had completed.

Closes #2859 from ashb/AIRFLOW-1897-view-logs-for-
running-instance

(cherry picked from commit 9731ce6fae85c09ede54e40883f61a9d960d99e0)
Signed-off-by: Bolke de Bruin <bolke@xs4all.nl>


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/a3baf9e7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/a3baf9e7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/a3baf9e7

Branch: refs/heads/v1-9-test
Commit: a3baf9e7f1d2ad410ad7a965f8c9946bb3244605
Parents: bc766e7
Author: Ash Berlin-Taylor <ash_github@firemirror.com>
Authored: Sat Dec 9 17:24:05 2017 +0100
Committer: Bolke de Bruin <bolke@xs4all.nl>
Committed: Sat Dec 9 22:06:01 2017 +0100

----------------------------------------------------------------------
 airflow/models.py                      |  4 +++
 airflow/utils/log/file_task_handler.py |  6 +++--
 tests/utils/test_log_handlers.py       | 40 +++++++++++++++++++++++++++++
 3 files changed, 48 insertions(+), 2 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/a3baf9e7/airflow/models.py
----------------------------------------------------------------------
diff --git a/airflow/models.py b/airflow/models.py
index e979b07..3e296eb 100755
--- a/airflow/models.py
+++ b/airflow/models.py
@@ -829,6 +829,10 @@ class TaskInstance(Base, LoggingMixin):
     def try_number(self, value):
         self._try_number = value
 
+    @property
+    def next_try_number(self):
+        return self._try_number + 1
+
     def command(
             self,
             mark_success=False,

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/a3baf9e7/airflow/utils/log/file_task_handler.py
----------------------------------------------------------------------
diff --git a/airflow/utils/log/file_task_handler.py b/airflow/utils/log/file_task_handler.py
index 9a8061a..2e40ce8 100644
--- a/airflow/utils/log/file_task_handler.py
+++ b/airflow/utils/log/file_task_handler.py
@@ -139,12 +139,14 @@ class FileTaskHandler(logging.Handler):
         # So the log for a particular task try will only show up when
         # try number gets incremented in DB, i.e logs produced the time
         # after cli run and before try_number + 1 in DB will not be displayed.
-        next_try = task_instance.try_number
 
         if try_number is None:
+            next_try = task_instance.next_try_number
             try_numbers = list(range(1, next_try))
         elif try_number < 1:
-            logs = ['Error fetching the logs. Try number {} is invalid.'.format(try_number)]
+            logs = [
+                'Error fetching the logs. Try number {} is invalid.'.format(try_number),
+            ]
             return logs
         else:
             try_numbers = [try_number]

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/a3baf9e7/tests/utils/test_log_handlers.py
----------------------------------------------------------------------
diff --git a/tests/utils/test_log_handlers.py b/tests/utils/test_log_handlers.py
index e64ed7c..3e94d31 100644
--- a/tests/utils/test_log_handlers.py
+++ b/tests/utils/test_log_handlers.py
@@ -26,6 +26,7 @@ from airflow.operators.python_operator import PythonOperator
 from airflow.settings import Session
 from airflow.utils.log.logging_mixin import set_context
 from airflow.utils.log.file_task_handler import FileTaskHandler
+from airflow.utils.state import State
 
 DEFAULT_DATE = datetime(2016, 1, 1)
 TASK_LOGGER = 'airflow.task'
@@ -110,6 +111,45 @@ class TestFileTaskLogHandler(unittest.TestCase):
         # Remove the generated tmp log file.
         os.remove(log_filename)
 
+    def test_file_task_handler_running(self):
+        def task_callable(ti, **kwargs):
+            ti.log.info("test")
+        dag = DAG('dag_for_testing_file_task_handler', start_date=DEFAULT_DATE)
+        task = PythonOperator(
+            task_id='task_for_testing_file_log_handler',
+            dag=dag,
+            python_callable=task_callable,
+            provide_context=True
+        )
+        ti = TaskInstance(task=task, execution_date=DEFAULT_DATE)
+        ti.try_number = 2
+        ti.state = State.RUNNING
+
+        logger = ti.log
+        ti.log.disabled = False
+
+        file_handler = next((handler for handler in logger.handlers
+                             if handler.name == FILE_TASK_HANDLER), None)
+        self.assertIsNotNone(file_handler)
+
+        set_context(logger, ti)
+        self.assertIsNotNone(file_handler.handler)
+        # We expect set_context generates a file locally.
+        log_filename = file_handler.handler.baseFilename
+        self.assertTrue(os.path.isfile(log_filename))
+        self.assertTrue(log_filename.endswith("2.log"), log_filename)
+
+        logger.info("Test")
+
+        # Return value of read must be a list.
+        logs = file_handler.read(ti)
+        self.assertTrue(isinstance(logs, list))
+        # Logs for running tasks should show up too.
+        self.assertEqual(len(logs), 2)
+
+        # Remove the generated tmp log file.
+        os.remove(log_filename)
+
 
 class TestFilenameRendering(unittest.TestCase):
 


Mime
View raw message