airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From "Dmytro Kulyk (JIRA)" <j...@apache.org>
Subject [jira] [Updated] (AIRFLOW-1783) DAG created from function become unavailable in UI sporadically
Date Sat, 04 Nov 2017 17:55:00 GMT

     [ https://issues.apache.org/jira/browse/AIRFLOW-1783?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]

Dmytro Kulyk updated AIRFLOW-1783:
----------------------------------
    Description: 
I have a DAG created from function (see codescript below)
After uploading to server it become visible and can be executed by scheduler or manually
[^Capture_before.JPG] 
[^Capture_tree.JPG]
!Capture_tree.jpg|thumbnail!
However, in case of any activity upon this object (refresh etc) there is an "kaboom error"
[^Capture_error.JPG]
[^Capture_after.JPG]
{code}
-------------------------------------------------------------------------------
Node: uat01.airflow.retail
-------------------------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1988, in wsgi_app
    response = self.full_dispatch_request()
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1641, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1544, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1639, in full_dispatch_request
    rv = self.dispatch_request()
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1625, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 69, in inner
    return self._run_view(f, *args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 368, in _run_view
    return fn(self, *args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in decorated_view
    return func(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line 219, in view_func
    return f(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line 125, in wrapper
    return f(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line 1161, in tree
    base_date = dag.latest_execution_date or datetime.now()
AttributeError: 'NoneType' object has no attribute 'latest_execution_date'
{code}

cube_update.py
{code}
from datetime import datetime
from lib.cube_update import cube_updater

env = 'sit'
start_date = datetime(2017, 7, 1)
schedule_interval = '0 8 * * *'
 
dag = cube_updater(env, start_date, schedule_interval)
{code}
lib/__init__.py - empty one
lib/cube_update.py
{code}
from airflow import DAG
...

def cube_updater(env, start_date, schedule_interval):
    ...
    dag = DAG(
        dag_id = dag_id,
        default_args = default_args,
        description = 'OLAP Cube updater',
        start_date = start_date, #datetime(2017, 9, 1),
        schedule_interval = schedule_interval, #'0 8 * * *',
        max_active_runs = 6,
        catchup=True)
....
    return dag
{code}


  was:
I have a DAG created from function (see codescript below)
After uploading to server it become visible and can be executed by scheduler or manually
[^Capture_before.JPG] [^Capture_tree.JPG]
However, in case of any activity upon this object (refresh etc) there is an "kaboom error"
[^Capture_error.JPG]
[^Capture_after.JPG]
{code}
-------------------------------------------------------------------------------
Node: uat01.airflow.retail
-------------------------------------------------------------------------------
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1988, in wsgi_app
    response = self.full_dispatch_request()
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1641, in full_dispatch_request
    rv = self.handle_user_exception(e)
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1544, in handle_user_exception
    reraise(exc_type, exc_value, tb)
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1639, in full_dispatch_request
    rv = self.dispatch_request()
  File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1625, in dispatch_request
    return self.view_functions[rule.endpoint](**req.view_args)
  File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 69, in inner
    return self._run_view(f, *args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 368, in _run_view
    return fn(self, *args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in decorated_view
    return func(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line 219, in view_func
    return f(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line 125, in wrapper
    return f(*args, **kwargs)
  File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line 1161, in tree
    base_date = dag.latest_execution_date or datetime.now()
AttributeError: 'NoneType' object has no attribute 'latest_execution_date'
{code}

cube_update.py
{code}
from datetime import datetime
from lib.cube_update import cube_updater

env = 'sit'
start_date = datetime(2017, 7, 1)
schedule_interval = '0 8 * * *'
 
dag = cube_updater(env, start_date, schedule_interval)
{code}
lib/__init__.py - empty one
lib/cube_update.py
{code}
from airflow import DAG
...

def cube_updater(env, start_date, schedule_interval):
    ...
    dag = DAG(
        dag_id = dag_id,
        default_args = default_args,
        description = 'OLAP Cube updater',
        start_date = start_date, #datetime(2017, 9, 1),
        schedule_interval = schedule_interval, #'0 8 * * *',
        max_active_runs = 6,
        catchup=True)
....
    return dag
{code}



> DAG created from function become unavailable in UI sporadically
> ---------------------------------------------------------------
>
>                 Key: AIRFLOW-1783
>                 URL: https://issues.apache.org/jira/browse/AIRFLOW-1783
>             Project: Apache Airflow
>          Issue Type: Bug
>          Components: DAG, scheduler, ui, webserver
>    Affects Versions: 1.8.1, 1.8.2
>         Environment: Ubuntu 16.04.3 LTS
> Python 2.7.12
> CeleryExecutor: 2-nodes cluster
>            Reporter: Dmytro Kulyk
>            Priority: Major
>         Attachments: Capture_after.JPG, Capture_before.JPG, Capture_error.JPG, Capture_tree.JPG,
cube_update.py, cube_update_sit.py
>
>
> I have a DAG created from function (see codescript below)
> After uploading to server it become visible and can be executed by scheduler or manually
> [^Capture_before.JPG] 
> [^Capture_tree.JPG]
> !Capture_tree.jpg|thumbnail!
> However, in case of any activity upon this object (refresh etc) there is an "kaboom error"
> [^Capture_error.JPG]
> [^Capture_after.JPG]
> {code}
> -------------------------------------------------------------------------------
> Node: uat01.airflow.retail
> -------------------------------------------------------------------------------
> Traceback (most recent call last):
>   File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1988, in wsgi_app
>     response = self.full_dispatch_request()
>   File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1641, in full_dispatch_request
>     rv = self.handle_user_exception(e)
>   File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1544, in handle_user_exception
>     reraise(exc_type, exc_value, tb)
>   File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1639, in full_dispatch_request
>     rv = self.dispatch_request()
>   File "/usr/local/lib/python2.7/dist-packages/flask/app.py", line 1625, in dispatch_request
>     return self.view_functions[rule.endpoint](**req.view_args)
>   File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 69, in inner
>     return self._run_view(f, *args, **kwargs)
>   File "/usr/local/lib/python2.7/dist-packages/flask_admin/base.py", line 368, in _run_view
>     return fn(self, *args, **kwargs)
>   File "/usr/local/lib/python2.7/dist-packages/flask_login.py", line 755, in decorated_view
>     return func(*args, **kwargs)
>   File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line 219, in view_func
>     return f(*args, **kwargs)
>   File "/usr/local/lib/python2.7/dist-packages/airflow/www/utils.py", line 125, in wrapper
>     return f(*args, **kwargs)
>   File "/usr/local/lib/python2.7/dist-packages/airflow/www/views.py", line 1161, in tree
>     base_date = dag.latest_execution_date or datetime.now()
> AttributeError: 'NoneType' object has no attribute 'latest_execution_date'
> {code}
> cube_update.py
> {code}
> from datetime import datetime
> from lib.cube_update import cube_updater
> env = 'sit'
> start_date = datetime(2017, 7, 1)
> schedule_interval = '0 8 * * *'
>  
> dag = cube_updater(env, start_date, schedule_interval)
> {code}
> lib/__init__.py - empty one
> lib/cube_update.py
> {code}
> from airflow import DAG
> ...
> def cube_updater(env, start_date, schedule_interval):
>     ...
>     dag = DAG(
>         dag_id = dag_id,
>         default_args = default_args,
>         description = 'OLAP Cube updater',
>         start_date = start_date, #datetime(2017, 9, 1),
>         schedule_interval = schedule_interval, #'0 8 * * *',
>         max_active_runs = 6,
>         catchup=True)
> ....
>     return dag
> {code}



--
This message was sent by Atlassian JIRA
(v6.4.14#64029)

Mime
View raw message