airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From davy...@apache.org
Subject incubator-airflow git commit: [AIRFLOW-218] Added option to enable webserver gunicorn access/err logs
Date Wed, 08 Jun 2016 05:14:26 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/master 80210b2bd -> 0bd5515a4


[AIRFLOW-218] Added option to enable webserver gunicorn access/err logs

Closes #1577 from aoen/ddavydov/better_http_response_logging

Added an option to enable gunicorn access/error logs.
The default config will now log webserver errors/accesses to stderr.

Also made the 404 page hostname text default page color instead of
white, since white text is pretty hard to see against a white
background.


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/0bd5515a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/0bd5515a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/0bd5515a

Branch: refs/heads/master
Commit: 0bd5515a42f7912b0d4ac8bf33dec2f01539b555
Parents: 80210b2
Author: Dan Davydov <dan.davydov@airbnb.com>
Authored: Tue Jun 7 22:13:48 2016 -0700
Committer: Dan Davydov <dan.davydov@airbnb.com>
Committed: Tue Jun 7 22:14:01 2016 -0700

----------------------------------------------------------------------
 airflow/bin/cli.py                         | 55 ++++++++++++++++++-------
 airflow/configuration.py                   | 10 ++++-
 airflow/www/templates/airflow/circles.html |  2 +-
 3 files changed, 49 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0bd5515a/airflow/bin/cli.py
----------------------------------------------------------------------
diff --git a/airflow/bin/cli.py b/airflow/bin/cli.py
index 87400eb..3430164 100755
--- a/airflow/bin/cli.py
+++ b/airflow/bin/cli.py
@@ -416,6 +416,8 @@ def webserver(args):
 
     from airflow.www.app import cached_app
     app = cached_app(conf)
+    access_logfile = args.access_logfile or conf.get('webserver', 'access_logfile')
+    error_logfile = args.error_logfile or conf.get('webserver', 'error_logfile')
     workers = args.workers or conf.get('webserver', 'workers')
     worker_timeout = (args.worker_timeout or
                       conf.get('webserver', 'webserver_worker_timeout'))
@@ -427,23 +429,36 @@ def webserver(args):
     else:
         pid, stdout, stderr, log_file = setup_locations("webserver", pid=args.pid)
         print(
-            'Running the Gunicorn server with {workers} {args.workerclass}'
-            'workers on host {args.hostname} and port '
-            '{args.port} with a timeout of {worker_timeout}...'.format(**locals()))
-
-        run_args = ['gunicorn',
-                    '-w ' + str(args.workers),
-                    '-k ' + str(args.workerclass),
-                    '-t ' + str(args.worker_timeout),
-                    '-b ' + args.hostname + ':' + str(args.port),
-                    '-n ' + 'airflow-webserver',
-                    '-p ' + str(pid)]
+            textwrap.dedent('''\
+                Running the Gunicorn Server with:
+                Workers: {workers} {args.workerclass}
+                Host: {args.hostname}:{args.port}
+                Timeout: {worker_timeout}
+                Logfiles: {access_logfile} {error_logfile}
+                =================================================================\
+            '''.format(**locals())))
+
+        run_args = [
+            'gunicorn',
+            '-w ' + str(args.workers),
+            '-k ' + str(args.workerclass),
+            '-t ' + str(args.worker_timeout),
+            '-b ' + args.hostname + ':' + str(args.port),
+            '-n ' + 'airflow-webserver',
+            '-p ' + str(pid),
+        ]
+
+        if args.access_logfile:
+            run_args += ['--access-logfile', str(args.access_logfile)]
+
+        if args.error_logfile:
+            run_args += ['--error-logfile', str(args.error_logfile)]
 
         if args.daemon:
-            run_args.append("-D")
+            run_args += ["-D"]
 
         module = "airflow.www.app:cached_app()".encode()
-        run_args.append(module)
+        run_args += [module]
         os.execvp(
             'gunicorn', run_args
         )
@@ -799,6 +814,16 @@ class CLIFactory(object):
             ("-d", "--debug"),
             "Use the server that ships with Flask in debug mode",
             "store_true"),
+        'access_logfile': Arg(
+            ("-A", "--access_logfile"),
+            default=conf.get('webserver', 'ACCESS_LOGFILE'),
+            help="The logfile to store the webserver access log. Use '-' to print to "
+                 "stderr."),
+        'error_logfile': Arg(
+            ("-E", "--error_logfile"),
+            default=conf.get('webserver', 'ERROR_LOGFILE'),
+            help="The logfile to store the webserver error log. Use '-' to print to "
+                 "stderr."),
         # resetdb
         'yes': Arg(
             ("-y", "--yes"),
@@ -925,8 +950,8 @@ class CLIFactory(object):
             'func': webserver,
             'help': "Start a Airflow webserver instance",
             'args': ('port', 'workers', 'workerclass', 'worker_timeout', 'hostname',
-                     'pid', 'daemon', 'stdout', 'stderr', 'log_file',
-                     'debug'),
+                     'pid', 'daemon', 'stdout', 'stderr', 'access_logfile',
+                     'error_logfile', 'log_file', 'debug'),
         }, {
             'func': resetdb,
             'help': "Burn down and rebuild the metadata database",

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0bd5515a/airflow/configuration.py
----------------------------------------------------------------------
diff --git a/airflow/configuration.py b/airflow/configuration.py
index 3193fbf..3403656 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -98,7 +98,7 @@ defaults = {
         'remote_base_log_folder': '',
         'remote_log_conn_id': '',
         'encrypt_s3_logs': False,
-        's3_log_folder': '', # deprecated!
+        's3_log_folder': '',  # deprecated!
         'dag_concurrency': 16,
         'max_active_runs_per_dag': 16,
         'executor': 'SequentialExecutor',
@@ -122,7 +122,9 @@ defaults = {
         'secret_key': 'airflowified',
         'expose_config': False,
         'workers': 4,
-        'worker_class': 'sync'
+        'worker_class': 'sync',
+        'access_logfile': '',
+        'error_logfile': '',
     },
     'scheduler': {
         'statsd_on': False,
@@ -277,6 +279,10 @@ workers = 4
 # sync (default), eventlet, gevent
 worker_class = sync
 
+# Log files for the gunicorn webserver. '-' means log to stderr.
+access_logfile = -
+error_logfile = -
+
 # Expose the configuration file in the web server
 expose_config = true
 

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/0bd5515a/airflow/www/templates/airflow/circles.html
----------------------------------------------------------------------
diff --git a/airflow/www/templates/airflow/circles.html b/airflow/www/templates/airflow/circles.html
index 44e1349..7407d8d 100644
--- a/airflow/www/templates/airflow/circles.html
+++ b/airflow/www/templates/airflow/circles.html
@@ -1,6 +1,6 @@
 <div style="font-family: verdana;">
     <h1>Airflow 404 = lots of circles</h1>
-    <div style="color: white">{{ hostname }}</div>
+    <div>{{ hostname }}</div>
 <div
             id="div_svg"
             class="content"


Mime
View raw message