airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From GitBox <...@apache.org>
Subject [GitHub] ashb closed pull request #2449: [AIRFLOW-1371] Customize log_filepath to allow files writen to NFS-shares
Date Thu, 01 Nov 2018 21:11:42 GMT
ashb closed pull request #2449: [AIRFLOW-1371] Customize log_filepath to allow files writen
to NFS-shares
URL: https://github.com/apache/incubator-airflow/pull/2449
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py
index 4b3a0edc8f..d5b83d6268 100755
--- a/airflow/bin/cli.py
+++ b/airflow/bin/cli.py
@@ -361,8 +361,9 @@ def run(args, dag=None):
             # Create the directory as globally writable using custom mkdirs
             # as os.makedirs doesn't set mode properly.
             mkdirs(directory, 0o775)
-        iso = args.execution_date.isoformat()
-        filename = "{directory}/{iso}".format(**locals())
+        pattern = conf.get('core', 'LOG_FILE_DATETIME_PATTERN')
+        log_file = args.execution_date.strftime(pattern)
+        filename = "{directory}/{log_file}".format(**locals())
 
         if not os.path.exists(filename):
             open(filename, "a").close()
diff --git a/airflow/config_templates/default_airflow.cfg b/airflow/config_templates/default_airflow.cfg
index ddd1ba816e..85fd83cf50 100644
--- a/airflow/config_templates/default_airflow.cfg
+++ b/airflow/config_templates/default_airflow.cfg
@@ -55,6 +55,11 @@ log_format_with_pid = [%%(asctime)s] [%%(process)d] {{%%(filename)s:%%(lineno)d}
 log_format_with_thread_name = [%%(asctime)s] {{%%(filename)s:%%(lineno)d}} %%(threadName)s
%%(levelname)s - %%(message)s
 simple_log_format = %%(asctime)s %%(levelname)s - %%(message)s
 
+# Log filename format
+# used by airflow to create task-runs log files based on the execution_date,
+# some filesystems like NFS have colon as a restricted char
+log_file_datetime_pattern = %%Y-%%m-%%dT%%H:%%M:%%S
+
 # The executor class that airflow should use. Choices include
 # SequentialExecutor, LocalExecutor, CeleryExecutor, DaskExecutor
 executor = SequentialExecutor
diff --git a/airflow/models.py b/airflow/models.py
index 32ad144a22..ae1c3b403c 100755
--- a/airflow/models.py
+++ b/airflow/models.py
@@ -939,10 +939,15 @@ def generate_command(dag_id,
 
     @property
     def log_filepath(self):
-        iso = self.execution_date.isoformat()
-        log = os.path.expanduser(configuration.get('core', 'BASE_LOG_FOLDER'))
-        return (
-            "{log}/{self.dag_id}/{self.task_id}/{iso}.log".format(**locals()))
+        base_log_path = os.path.expanduser(
+            configuration.get('core', 'BASE_LOG_FOLDER')
+        )
+        pattern = configuration.get('core', 'LOG_FILE_DATETIME_PATTERN')
+        log_file = self.execution_date.strftime(pattern)
+        return ("{base_log_path}/"
+                "{self.dag_id}/"
+                "{self.task_id}/"
+                "{log_file}.log").format(**locals())
 
     @property
     def log_url(self):
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 6c39462072..fb2f56274a 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -700,17 +700,20 @@ def rendered(self):
     def log(self):
         BASE_LOG_FOLDER = os.path.expanduser(
             conf.get('core', 'BASE_LOG_FOLDER'))
+        pattern = conf.get('core', 'LOG_FILE_DATETIME_PATTERN')
+
         dag_id = request.args.get('dag_id')
         task_id = request.args.get('task_id')
         execution_date = request.args.get('execution_date')
+
+        dttm = dateutil.parser.parse(execution_date)
+        log_filename = dttm.strftime(pattern)
+        log_relative = "{dag_id}/{task_id}/{log_filename}".format(**locals())
         dag = dagbag.get_dag(dag_id)
-        log_relative = "{dag_id}/{task_id}/{execution_date}".format(
-            **locals())
         loc = os.path.join(BASE_LOG_FOLDER, log_relative)
         loc = loc.format(**locals())
         log = ""
         TI = models.TaskInstance
-        dttm = dateutil.parser.parse(execution_date)
         form = DateTimeForm(data={'execution_date': dttm})
         session = Session()
         ti = session.query(TI).filter(


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

Mime
View raw message