airflow-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From davy...@apache.org
Subject incubator-airflow git commit: [AIRFLOW-1519] Add server side paging in DAGs list
Date Fri, 15 Sep 2017 23:41:58 GMT
Repository: incubator-airflow
Updated Branches:
  refs/heads/master 6632b0ce1 -> b6d2e0a46


[AIRFLOW-1519] Add server side paging in DAGs list

Airflow's main page previously did paging client-
side via a
jQuery plugin (DataTable) which was very slow at
loading all DAGs.
The browser would load all DAGs in the table.
The result was performance degradation when having
a number of
DAGs in the range of 1K.

This commit implements server-side paging using
the webserver page
size setting, sending to the browser only the
elements for the
specific page.

Closes #2531 from edgarRd/erod-ui-dags-paging


Project: http://git-wip-us.apache.org/repos/asf/incubator-airflow/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-airflow/commit/b6d2e0a4
Tree: http://git-wip-us.apache.org/repos/asf/incubator-airflow/tree/b6d2e0a4
Diff: http://git-wip-us.apache.org/repos/asf/incubator-airflow/diff/b6d2e0a4

Branch: refs/heads/master
Commit: b6d2e0a46978e93e16576604624f57d1388814f2
Parents: 6632b0c
Author: Edgar Rodriguez <edgar.rodriguez@airbnb.com>
Authored: Fri Sep 15 16:41:25 2017 -0700
Committer: Dan Davydov <dan.davydov@airbnb.com>
Committed: Fri Sep 15 16:41:29 2017 -0700

----------------------------------------------------------------------
 airflow/www/static/bootstrap3-typeahead.min.js |  21 ++++
 airflow/www/templates/airflow/dags.html        |  76 +++++++++++-
 airflow/www/utils.py                           | 121 ++++++++++++++++++++
 airflow/www/views.py                           | 105 +++++++++++++----
 licenses/LICENSE-typeahead.txt                 |  13 +++
 5 files changed, 308 insertions(+), 28 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/b6d2e0a4/airflow/www/static/bootstrap3-typeahead.min.js
----------------------------------------------------------------------
diff --git a/airflow/www/static/bootstrap3-typeahead.min.js b/airflow/www/static/bootstrap3-typeahead.min.js
new file mode 100644
index 0000000..23aac4e
--- /dev/null
+++ b/airflow/www/static/bootstrap3-typeahead.min.js
@@ -0,0 +1,21 @@
+/* =============================================================
+ * bootstrap3-typeahead.js v4.0.2
+ * https://github.com/bassjobsen/Bootstrap-3-Typeahead
+ * =============================================================
+ * Original written by @mdo and @fat
+ * =============================================================
+ * Copyright 2014 Bass Jobsen @bassjobsen
+ *
+ * Licensed under the Apache License, Version 2.0 (the 'License');
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an 'AS IS' BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ * ============================================================ */
+!function(a,b){"use strict";"undefined"!=typeof module&&module.exports?module.exports=b(require("jquery")):"function"==typeof
define&&define.amd?define(["jquery"],function(a){return b(a)}):b(a.jQuery)}(this,function(a){"use
strict";var b=function(c,d){this.$element=a(c),this.options=a.extend({},b.defaults,d),this.matcher=this.options.matcher||this.matcher,this.sorter=this.options.sorter||this.sorter,this.select=this.options.select||this.select,this.autoSelect="boolean"!=typeof
this.options.autoSelect||this.options.autoSelect,this.highlighter=this.options.highlighter||this.highlighter,this.render=this.options.render||this.render,this.updater=this.options.updater||this.updater,this.displayText=this.options.displayText||this.displayText,this.source=this.options.source,this.delay=this.options.delay,this.$menu=a(this.options.menu),this.$appendTo=this.options.appendTo?a(this.options.appendTo):null,this.fitToElement="boolean"==typeof
this.options.fitToElement&&this.options.fitToElement,thi
 s.shown=!1,this.listen(),this.showHintOnFocus=("boolean"==typeof this.options.showHintOnFocus||"all"===this.options.showHintOnFocus)&&this.options.showHintOnFocus,this.afterSelect=this.options.afterSelect,this.addItem=!1,this.value=this.$element.val()||this.$element.text()};b.prototype={constructor:b,select:function(){var
a=this.$menu.find(".active").data("value");if(this.$element.data("active",a),this.autoSelect||a){var
b=this.updater(a);b||(b=""),this.$element.val(this.displayText(b)||b).text(this.displayText(b)||b).change(),this.afterSelect(b)}return
this.hide()},updater:function(a){return a},setSource:function(a){this.source=a},show:function(){var
d,b=a.extend({},this.$element.position(),{height:this.$element[0].offsetHeight}),c="function"==typeof
this.options.scrollHeight?this.options.scrollHeight.call():this.options.scrollHeight;if(this.shown?d=this.$menu:this.$appendTo?(d=this.$menu.appendTo(this.$appendTo),this.hasSameParent=this.$appendTo.is(this.$element.parent())):(d=this
 .$menu.insertAfter(this.$element),this.hasSameParent=!0),!this.hasSameParent){d.css("position","fixed");var
e=this.$element.offset();b.top=e.top,b.left=e.left}var f=a(d).parent().hasClass("dropup"),g=f?"auto":b.top+b.height+c,h=a(d).hasClass("dropdown-menu-right"),i=h?"auto":b.left;return
d.css({top:g,left:i}).show(),this.options.fitToElement===!0&&d.css("width",this.$element.outerWidth()+"px"),this.shown=!0,this},hide:function(){return
this.$menu.hide(),this.shown=!1,this},lookup:function(b){if("undefined"!=typeof b&&null!==b?this.query=b:this.query=this.$element.val()||this.$element.text()||"",this.query.length<this.options.minLength&&!this.options.showHintOnFocus)return
this.shown?this.hide():this;var d=a.proxy(function(){a.isFunction(this.source)?this.source(this.query,a.proxy(this.process,this)):this.source&&this.process(this.source)},this);clearTimeout(this.lookupWorker),this.lookupWorker=setTimeout(d,this.delay)},process:function(b){var
c=this;return b=a.grep(b,function(a){re
 turn c.matcher(a)}),b=this.sorter(b),b.length||this.options.addItem?(b.length>0?this.$element.data("active",b[0]):this.$element.data("active",null),this.options.addItem&&b.push(this.options.addItem),"all"==this.options.items?this.render(b).show():this.render(b.slice(0,this.options.items)).show()):this.shown?this.hide():this},matcher:function(a){var
b=this.displayText(a);return~b.toLowerCase().indexOf(this.query.toLowerCase())},sorter:function(a){for(var
e,b=[],c=[],d=[];e=a.shift();){var f=this.displayText(e);f.toLowerCase().indexOf(this.query.toLowerCase())?~f.indexOf(this.query)?c.push(e):d.push(e):b.push(e)}return
b.concat(c,d)},highlighter:function(a){var b=this.query;if(""===b)return a;var f,c=a.match(/(>)([^<]*)(<)/g),d=[],e=[];if(c&&c.length)for(f=0;f<c.length;++f)c[f].length>2&&d.push(c[f]);else
d=[],d.push(a);b = b.replace((/[\(\)\/\.\*\+\?\[\]]/g), function(m) {return '\\'+m;});var
h,g=new RegExp(b,"g");for(f=0;f<d.length;++f)h=d[f].match(g),h&&h.length>0&&e.push(d[f]);for
 (f=0;f<e.length;++f)a=a.replace(e[f],e[f].replace(g,"<strong>$&</strong>"));return
a},render:function(b){var c=this,d=this,e=!1,f=[],g=c.options.separator;return a.each(b,function(a,c){a>0&&c[g]!==b[a-1][g]&&f.push({__type:"divider"}),!c[g]||0!==a&&c[g]===b[a-1][g]||f.push({__type:"category",name:c[g]}),f.push(c)}),b=a(f).map(function(b,f){if("category"==(f.__type||!1))return
a(c.options.headerHtml).text(f.name)[0];if("divider"==(f.__type||!1))return a(c.options.headerDivider)[0];var
g=d.displayText(f);return b=a(c.options.item).data("value",f),b.find("a").html(c.highlighter(g,f)),g==d.$element.val()&&(b.addClass("active"),d.$element.data("active",f),e=!0),b[0]}),this.autoSelect&&!e&&(b.filter(":not(.dropdown-header)").first().addClass("active"),this.$element.data("active",b.first().data("value"))),this.$menu.html(b),this},displayText:function(a){return"undefined"!=typeof
a&&"undefined"!=typeof a.name?a.name:a},next:function(b){var c=this.$menu.find(".active").removeClass("active"),
 d=c.next();d.length||(d=a(this.$menu.find("li")[0])),d.addClass("active")},prev:function(a){var
b=this.$menu.find(".active").removeClass("active"),c=b.prev();c.length||(c=this.$menu.find("li").last()),c.addClass("active")},listen:function(){this.$element.on("focus",a.proxy(this.focus,this)).on("blur",a.proxy(this.blur,this)).on("keypress",a.proxy(this.keypress,this)).on("input",a.proxy(this.input,this)).on("keyup",a.proxy(this.keyup,this)),this.eventSupported("keydown")&&this.$element.on("keydown",a.proxy(this.keydown,this)),this.$menu.on("click",a.proxy(this.click,this)).on("mouseenter","li",a.proxy(this.mouseenter,this)).on("mouseleave","li",a.proxy(this.mouseleave,this)).on("mousedown",a.proxy(this.mousedown,this))},destroy:function(){this.$element.data("typeahead",null),this.$element.data("active",null),this.$element.off("focus").off("blur").off("keypress").off("input").off("keyup"),this.eventSupported("keydown")&&this.$element.off("keydown"),this.$menu.remove(),this.destroyed=!
 0},eventSupported:function(a){var b=a in this.$element;return b||(this.$element.setAttribute(a,"return;"),b="function"==typeof
this.$element[a]),b},move:function(a){if(this.shown)switch(a.keyCode){case 9:case 13:case
27:a.preventDefault();break;case 38:if(a.shiftKey)return;a.preventDefault(),this.prev();break;case
40:if(a.shiftKey)return;a.preventDefault(),this.next()}},keydown:function(b){this.suppressKeyPressRepeat=~a.inArray(b.keyCode,[40,38,9,13,27]),this.shown||40!=b.keyCode?this.move(b):this.lookup()},keypress:function(a){this.suppressKeyPressRepeat||this.move(a)},input:function(a){var
b=this.$element.val()||this.$element.text();this.value!==b&&(this.value=b,this.lookup())},keyup:function(a){if(!this.destroyed)switch(a.keyCode){case
40:case 38:case 16:case 17:case 18:break;case 9:case 13:if(!this.shown)return;this.select();break;case
27:if(!this.shown)return;this.hide()}},focus:function(a){this.focused||(this.focused=!0,this.options.showHintOnFocus&&this.skipShowHintOnFocus!==
 !0&&("all"===this.options.showHintOnFocus?this.lookup(""):this.lookup())),this.skipShowHintOnFocus&&(this.skipShowHintOnFocus=!1)},blur:function(a){this.mousedover||this.mouseddown||!this.shown?this.mouseddown&&(this.skipShowHintOnFocus=!0,this.$element.focus(),this.mouseddown=!1):(this.hide(),this.focused=!1)},click:function(a){a.preventDefault(),this.skipShowHintOnFocus=!0,this.select(),this.$element.focus(),this.hide()},mouseenter:function(b){this.mousedover=!0,this.$menu.find(".active").removeClass("active"),a(b.currentTarget).addClass("active")},mouseleave:function(a){this.mousedover=!1,!this.focused&&this.shown&&this.hide()},mousedown:function(a){this.mouseddown=!0,this.$menu.one("mouseup",function(a){this.mouseddown=!1}.bind(this))}};var
c=a.fn.typeahead;a.fn.typeahead=function(c){var d=arguments;return"string"==typeof c&&"getActive"==c?this.data("active"):this.each(function(){var
e=a(this),f=e.data("typeahead"),g="object"==typeof c&&c;f||e.data("typeahead",f=new
b(this,g)),"
 string"==typeof c&&f[c]&&(d.length>1?f[c].apply(f,Array.prototype.slice.call(d,1)):f[c]())})},b.defaults={source:[],items:8,menu:'<ul
class="typeahead dropdown-menu" role="listbox"></ul>',item:'<li><a class="dropdown-item"
href="#" role="option"></a></li>',minLength:1,scrollHeight:0,autoSelect:!0,afterSelect:a.noop,addItem:!1,delay:0,separator:"category",headerHtml:'<li
class="dropdown-header"></li>',headerDivider:'<li class="divider" role="separator"></li>'},a.fn.typeahead.Constructor=b,a.fn.typeahead.noConflict=function(){return
a.fn.typeahead=c,this},a(document).on("focus.typeahead.data-api",'[data-provide="typeahead"]',function(b){var
c=a(this);c.data("typeahead")||c.typeahead(c.data())})});

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/b6d2e0a4/airflow/www/templates/airflow/dags.html
----------------------------------------------------------------------
diff --git a/airflow/www/templates/airflow/dags.html b/airflow/www/templates/airflow/dags.html
index 513e4aa..098ccb9 100644
--- a/airflow/www/templates/airflow/dags.html
+++ b/airflow/www/templates/airflow/dags.html
@@ -29,11 +29,23 @@
   <h2>DAGs</h2>
 
   <div id="main_content" style="display:none;">
+    <div class="row">
+      <div class="col-sm-2">
+      </div>
+      <div class="col-sm-10">
+        <form id="search_form" class="form-inline" style="width: 100%; text-align: right;">
+            <div id="dags_filter" class="form-group" style="width: 100%;">
+              <label for="dag_query" style="width:20%; text-align: right;">Search:</label>
+              <input id="dag_query" type="text" class="typeahead form-control" data-provide="typeahead"
style="width:50%;" value="{{search_query}}">
+            </div>
+        </form>
+      </div>
+    </div>
     <table id="dags" class="table table-striped table-bordered">
         <thead>
             <tr>
                 <th></th>
-                <th width="12"><span id="pause_header"class="glyphicon glyphicon-info-sign"
title="Use this toggle to pause a DAG. The scheduler won't schedule new tasks instances for
a paused DAG. Tasks already running at pause time won't be affected."></span></th>
+                <th width="12"><span id="pause_header" class="glyphicon glyphicon-info-sign"
title="Use this toggle to pause a DAG. The scheduler won't schedule new tasks instances for
a paused DAG. Tasks already running at pause time won't be affected."></span></th>
                 <th>DAG</th>
                 <th>Schedule</th>
                 <th>Owner</th>
@@ -51,7 +63,7 @@
             </tr>
         </thead>
         <tbody>
-        {% for dag_id in all_dag_ids %}
+        {% for dag_id in dag_ids_in_page %}
             {% set dag = webserver_dags[dag_id] if dag_id in webserver_dags else None %}
             <tr>
                 <!-- Column 1: Edit dag -->
@@ -174,6 +186,19 @@
         {% endfor %}
         </tbody>
     </table>
+    <div class="row">
+      <div class="col-sm-12" style="text-align:right;">
+        <div class="dataTables_info" id="dags_info" role="status" aria-live="polite" style="padding-top:
0px;">Showing {{num_dag_from}} to {{num_dag_to}} of {{num_of_all_dags}} entries</div>
+      </div>
+    </div>
+    <div class="row">
+      <div class="col-sm-12" style="text-align:left;">
+        <div class="dataTables_info" id="dags_paginate">
+          {{paging}}
+        </div>
+      </div>
+
+    </div>
     {% if not hide_paused %}
     <a href="/admin/?showPaused=False">Hide Paused DAGs</a>
     {% else %}
@@ -187,8 +212,26 @@
   <script src="{{ url_for('static', filename='d3.v3.min.js') }}"></script>
   <script src="{{ url_for('static', filename='jquery.dataTables.min.js') }}"></script>
   <script src="{{ url_for('static', filename='bootstrap-toggle.min.js') }}"></script>
+  <script src="{{ url_for('static', filename='bootstrap3-typeahead.min.js') }}"></script>
   <script>
 
+      const DAGS_INDEX = {{ url_for('admin.index') }}
+      const ENTER_KEY_CODE = 13
+
+      $('#dag_query').on('keypress', function (e) {
+        // check for key press on ENTER (key code 13) to trigger the search
+        if (e.which === ENTER_KEY_CODE) {
+          search_query = $('#dag_query').val();
+          window.location = DAGS_INDEX + "?search="+ encodeURI(search_query);
+          e.preventDefault();
+        }
+      });
+
+      $('#page_size').on('change', function() {
+        p_size = $(this).val();
+        window.location = DAGS_INDEX + "?page_size=" + p_size;
+      });
+
       function confirmTriggerDag(dag_id){
           return confirm("Are you sure you want to run '"+dag_id+"' now?");
       }
@@ -205,10 +248,37 @@
           $.post(url);
         });
       });
+
+      var $input = $(".typeahead");
+      unique_options_search = new Set([
+          {% for token in auto_complete_data %}
+            "{{token}}",
+          {% endfor %}
+        ]);
+
+      $input.typeahead({
+        source: [...unique_options_search],
+        autoSelect: false,
+        afterSelect: function(value) {
+          search_query = value.trim()
+          if (search_query) {
+            window.location = DAGS_INDEX + "?search="+ encodeURI(search_query);
+          }
+        }
+      });
+
+      $input.change(function() {
+        var current = $input.typeahead("getActive");
+
+      });
+
       $('#dags').dataTable({
         "iDisplayLength": 500,
         "bSort": false,
-        "pageLength": 25,
+        "searching": false,
+        "ordering": false,
+        "paging": false,
+        "info": false
       });
       $("#main_content").show(250);
       diameter = 25;

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/b6d2e0a4/airflow/www/utils.py
----------------------------------------------------------------------
diff --git a/airflow/www/utils.py b/airflow/www/utils.py
index 0846542..344a4e9 100644
--- a/airflow/www/utils.py
+++ b/airflow/www/utils.py
@@ -76,6 +76,127 @@ class DataProfilingMixin(object):
         )
 
 
+def generate_pages(current_page, num_of_pages,
+                   search=None, showPaused=None, window=7):
+    """
+    Generates the HTML for a paging component using a similar logic to the paging
+    auto-generated by Flask managed views. The paging component defines a number of
+    pages visible in the pager (window) and once the user goes to a page beyond the
+    largest visible, it would scroll to the right the page numbers and keeps the
+    current one in the middle of the pager component. When in the last pages,
+    the pages won't scroll and just keep moving until the last page. Pager also contains
+    <first, previous, ..., next, last> pages.
+    This component takes into account custom parameters such as search and showPaused,
+    which could be added to the pages link in order to maintain the state between
+    client and server. It also allows to make a bookmark on a specific paging state.
+    :param current_page:
+        the current page number, 0-indexed
+    :param num_of_pages:
+        the total number of pages
+    :param search:
+        the search query string, if any
+    :param showPaused:
+        false if paused dags will be hidden, otherwise true to show them
+    :param window:
+        the number of pages to be shown in the paging component (7 default)
+    :return:
+        the HTML string of the paging component
+    """
+
+    def get_params(**kwargs):
+        params = []
+        for k, v in kwargs.items():
+            if k == 'showPaused':
+                # True is default or None
+                if v or v is None:
+                    continue
+                params.append('{}={}'.format(k, v))
+            elif v:
+                params.append('{}={}'.format(k, v))
+        return '&'.join(params)
+
+    void_link = 'javascript:void(0)'
+    first_node = """<li class="paginate_button {disabled}" id="dags_first">
+    <a href="{href_link}" aria-controls="dags" data-dt-idx="0" tabindex="0">&laquo;</a>
+</li>"""
+
+    previous_node = """<li class="paginate_button previous {disabled}" id="dags_previous">
+    <a href="{href_link}" aria-controls="dags" data-dt-idx="0" tabindex="0">&lt;</a>
+</li>"""
+
+    next_node = """<li class="paginate_button next {disabled}" id="dags_next">
+    <a href="{href_link}" aria-controls="dags" data-dt-idx="3" tabindex="0">&gt;</a>
+</li>"""
+
+    last_node = """<li class="paginate_button {disabled}" id="dags_last">
+    <a href="{href_link}" aria-controls="dags" data-dt-idx="3" tabindex="0">&raquo;</a>
+</li>"""
+
+    page_node = """<li class="paginate_button {is_active}">
+    <a href="{href_link}" aria-controls="dags" data-dt-idx="2" tabindex="0">{page_num}</a>
+</li>"""
+
+    output = ['<ul class="pagination" style="margin-top:0px;">']
+
+    is_disabled = 'disabled' if current_page <= 0 else ''
+    output.append(first_node.format(href_link="?{}"
+                                    .format(get_params(page=0,
+                                                       search=search,
+                                                       showPaused=showPaused)),
+                                    disabled=is_disabled))
+
+    page_link = void_link
+    if current_page > 0:
+        page_link = '?{}'.format(get_params(page=(current_page - 1),
+                                            search=search,
+                                            showPaused=showPaused))
+
+    output.append(previous_node.format(href_link=page_link,
+                                       disabled=is_disabled))
+
+    mid = int(window / 2)
+    last_page = num_of_pages - 1
+
+    if current_page <= mid or num_of_pages < window:
+        pages = [i for i in range(0, min(num_of_pages, window))]
+    elif mid < current_page < last_page - mid:
+        pages = [i for i in range(current_page - mid, current_page + mid + 1)]
+    else:
+        pages = [i for i in range(num_of_pages - window, last_page + 1)]
+
+    def is_current(current, page):
+        return page == current
+
+    for page in pages:
+        vals = {
+            'is_active': 'active' if is_current(current_page, page) else '',
+            'href_link': void_link if is_current(current_page, page)
+                         else '?{}'.format(get_params(page=page,
+                                                      search=search,
+                                                      showPaused=showPaused)),
+            'page_num': page + 1
+        }
+        output.append(page_node.format(**vals))
+
+    is_disabled = 'disabled' if current_page >= num_of_pages - 1 else ''
+
+    page_link = (void_link if current_page >= num_of_pages - 1
+                 else '?{}'.format(get_params(page=current_page + 1,
+                                              search=search,
+                                              showPaused=showPaused)))
+
+    output.append(next_node.format(href_link=page_link, disabled=is_disabled))
+    output.append(last_node.format(href_link="?{}"
+                                   .format(get_params(page=last_page,
+                                                      search=search,
+                                                      showPaused=showPaused)),
+                                   disabled=is_disabled))
+
+    output.append('</ul>')
+
+    return wtforms.widgets.core.HTMLString('\n'.join(output))
+
+
 def limit_sql(sql, limit, conn_type):
     sql = sql.strip()
     sql = sql.rstrip(';')

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/b6d2e0a4/airflow/www/views.py
----------------------------------------------------------------------
diff --git a/airflow/www/views.py b/airflow/www/views.py
index 447c19f..850db4a 100644
--- a/airflow/www/views.py
+++ b/airflow/www/views.py
@@ -24,6 +24,7 @@ from functools import wraps
 from datetime import datetime, timedelta
 import dateutil.parser
 import copy
+import math
 import json
 import bleach
 from collections import defaultdict
@@ -222,10 +223,7 @@ attr_renderer = {
 
 
 def data_profiling_required(f):
-    '''
-    Decorator for views requiring data profiling access
-    '''
-
+    """Decorator for views requiring data profiling access"""
     @wraps(f)
     def decorated_function(*args, **kwargs):
         if (
@@ -309,9 +307,10 @@ class Airflow(BaseView):
         session.commit()
         session.close()
 
-        payload = {}
-        payload['state'] = 'ERROR'
-        payload['error'] = ''
+        payload = {
+            "state": "ERROR",
+            "error": ""
+        }
 
         # Processing templated fields
         try:
@@ -1786,7 +1785,6 @@ class HomeView(AdminIndexView):
     def index(self):
         session = Session()
         DM = models.DagModel
-        qry = None
 
         # restrict the dags shown if filter_by_owner and current user is not superuser
         do_filter = FILTER_BY_OWNER and (not current_user.is_superuser())
@@ -1795,40 +1793,52 @@ class HomeView(AdminIndexView):
         hide_paused_dags_by_default = conf.getboolean('webserver',
                                                       'hide_paused_dags_by_default')
         show_paused_arg = request.args.get('showPaused', 'None')
+
+        def get_int_arg(value, default=0):
+            try:
+                return int(value)
+            except ValueError:
+                return default
+
+        arg_current_page = request.args.get('page', '0')
+        arg_search_query = request.args.get('search', None)
+
+        dags_per_page = PAGE_SIZE
+        current_page = get_int_arg(arg_current_page, default=0)
+
         if show_paused_arg.strip().lower() == 'false':
             hide_paused = True
-
         elif show_paused_arg.strip().lower() == 'true':
             hide_paused = False
-
         else:
             hide_paused = hide_paused_dags_by_default
 
         # read orm_dags from the db
-        qry = session.query(DM)
-        qry_fltr = []
+        sql_query = session.query(DM)
 
         if do_filter and owner_mode == 'ldapgroup':
-            qry_fltr = qry.filter(
-                ~DM.is_subdag, DM.is_active,
+            sql_query = sql_query.filter(
+                ~DM.is_subdag,
+                DM.is_active,
                 DM.owners.in_(current_user.ldap_groups)
-            ).all()
+            )
         elif do_filter and owner_mode == 'user':
-            qry_fltr = qry.filter(
+            sql_query = sql_query.filter(
                 ~DM.is_subdag, DM.is_active,
                 DM.owners == current_user.user.username
-            ).all()
+            )
         else:
-            qry_fltr = qry.filter(
+            sql_query = sql_query.filter(
                 ~DM.is_subdag, DM.is_active
-            ).all()
+            )
 
         # optionally filter out "paused" dags
         if hide_paused:
-            orm_dags = {dag.dag_id: dag for dag in qry_fltr if not dag.is_paused}
+            sql_query = sql_query.filter(~DM.is_paused)
 
-        else:
-            orm_dags = {dag.dag_id: dag for dag in qry_fltr}
+        orm_dags = {dag.dag_id: dag for dag
+                    in sql_query
+                    .all()}
 
         import_errors = session.query(models.ImportError).all()
         for ie in import_errors:
@@ -1870,13 +1880,58 @@ class HomeView(AdminIndexView):
                 for dag in unfiltered_webserver_dags
             }
 
-        all_dag_ids = sorted(set(orm_dags.keys()) | set(webserver_dags.keys()))
+        if arg_search_query:
+            lower_search_query = arg_search_query.lower()
+            # filter by dag_id
+            webserver_dags_filtered = {
+                dag_id: dag
+                for dag_id, dag in webserver_dags.items()
+                if (lower_search_query in dag_id.lower() or
+                    lower_search_query in dag.owner.lower())
+            }
+
+            all_dag_ids = (set([dag.dag_id for dag in orm_dags.values()
+                                if lower_search_query in dag.dag_id.lower() or
+                                lower_search_query in dag.owners.lower()]) |
+                           set(webserver_dags_filtered.keys()))
+
+            sorted_dag_ids = sorted(all_dag_ids)
+        else:
+            webserver_dags_filtered = webserver_dags
+            sorted_dag_ids = sorted(set(orm_dags.keys()) | set(webserver_dags.keys()))
+
+        start = current_page * dags_per_page
+        end = start + dags_per_page
+
+        num_of_all_dags = len(sorted_dag_ids)
+        page_dag_ids = sorted_dag_ids[start:end]
+        num_of_pages = int(math.ceil(num_of_all_dags / float(dags_per_page)))
+
+        auto_complete_data = set()
+        for dag in webserver_dags_filtered.values():
+            auto_complete_data.add(dag.dag_id)
+            auto_complete_data.add(dag.owner)
+        for dag in orm_dags.values():
+            auto_complete_data.add(dag.dag_id)
+            auto_complete_data.add(dag.owners)
+
         return self.render(
             'airflow/dags.html',
-            webserver_dags=webserver_dags,
+            webserver_dags=webserver_dags_filtered,
             orm_dags=orm_dags,
             hide_paused=hide_paused,
-            all_dag_ids=all_dag_ids)
+            current_page=current_page,
+            search_query=arg_search_query if arg_search_query else '',
+            page_size=dags_per_page,
+            num_of_pages=num_of_pages,
+            num_dag_from=start + 1,
+            num_dag_to=min(end, num_of_all_dags),
+            num_of_all_dags=num_of_all_dags,
+            paging=wwwutils.generate_pages(current_page, num_of_pages,
+                                           search=arg_search_query,
+                                           showPaused=not hide_paused),
+            dag_ids_in_page=page_dag_ids,
+            auto_complete_data=auto_complete_data)
 
 
 class QueryView(wwwutils.DataProfilingMixin, BaseView):

http://git-wip-us.apache.org/repos/asf/incubator-airflow/blob/b6d2e0a4/licenses/LICENSE-typeahead.txt
----------------------------------------------------------------------
diff --git a/licenses/LICENSE-typeahead.txt b/licenses/LICENSE-typeahead.txt
new file mode 100644
index 0000000..0754b39
--- /dev/null
+++ b/licenses/LICENSE-typeahead.txt
@@ -0,0 +1,13 @@
+Copyright 2014 Bass Jobsen @bassjobsen
+
+Licensed under the Apache License, Version 2.0 (the 'License');
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an 'AS IS' BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.


Mime
View raw message