Mailing-List: contact commits-help@ambari.apache.org; run by ezmlm
Precedence: bulk
Reply-To: ambari-dev@ambari.apache.org
Content-Type: text/plain; charset="us-ascii"
MIME-Version: 1.0
Content-Transfer-Encoding: 8bit
From: aonishuk@apache.org
To: commits@ambari.apache.org
Date: Thu, 04 Sep 2014 13:07:09 -0000
Message-Id: <c355f6fcd0444fe28aecfebd1c9a5c3c@git.apache.org>
In-Reply-To: <ddac02666c37472894d0f6a8de2e2f69@git.apache.org>
References: <ddac02666c37472894d0f6a8de2e2f69@git.apache.org>
Subject: [18/22] AMBARI-7138. Ambari RPM deals with jinja2 dependency
 incorrectly (aonishuk)

http://git-wip-us.apache.org/repos/asf/ambari/blob/658360a5/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/filters.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/filters.py b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/filters.py
new file mode 100644
index 0000000..bdb545e
--- /dev/null
+++ b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/filters.py
@@ -0,0 +1,719 @@
+# -*- coding: utf-8 -*-
+"""
+    ambari_jinja2.filters
+    ~~~~~~~~~~~~~~
+
+    Bundled jinja filters.
+
+    :copyright: (c) 2010 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+import re
+import math
+from random import choice
+from operator import itemgetter
+from itertools import imap, groupby
+from ambari_jinja2.utils import Markup, escape, pformat, urlize, soft_unicode
+from ambari_jinja2.runtime import Undefined
+from ambari_jinja2.exceptions import FilterArgumentError, SecurityError
+
+
+_word_re = re.compile(r'\w+(?u)')
+
+
+def contextfilter(f):
+    """Decorator for marking context dependent filters. The current
+    :class:`Context` will be passed as first argument.
+    """
+    f.contextfilter = True
+    return f
+
+
+def evalcontextfilter(f):
+    """Decorator for marking eval-context dependent filters.  An eval
+    context object is passed as first argument.  For more information
+    about the eval context, see :ref:`eval-context`.
+
+    .. versionadded:: 2.4
+    """
+    f.evalcontextfilter = True
+    return f
+
+
+def environmentfilter(f):
+    """Decorator for marking evironment dependent filters.  The current
+    :class:`Environment` is passed to the filter as first argument.
+    """
+    f.environmentfilter = True
+    return f
+
+
+def do_forceescape(value):
+    """Enforce HTML escaping.  This will probably double escape variables."""
+    if hasattr(value, '__html__'):
+        value = value.__html__()
+    return escape(unicode(value))
+
+
+@evalcontextfilter
+def do_replace(eval_ctx, s, old, new, count=None):
+    """Return a copy of the value with all occurrences of a substring
+    replaced with a new one. The first argument is the substring
+    that should be replaced, the second is the replacement string.
+    If the optional third argument ``count`` is given, only the first
+    ``count`` occurrences are replaced:
+
+    .. sourcecode:: jinja
+
+        {{ "Hello World"|replace("Hello", "Goodbye") }}
+            -> Goodbye World
+
+        {{ "aaaaargh"|replace("a", "d'oh, ", 2) }}
+            -> d'oh, d'oh, aaargh
+    """
+    if count is None:
+        count = -1
+    if not eval_ctx.autoescape:
+        return unicode(s).replace(unicode(old), unicode(new), count)
+    if hasattr(old, '__html__') or hasattr(new, '__html__') and \
+       not hasattr(s, '__html__'):
+        s = escape(s)
+    else:
+        s = soft_unicode(s)
+    return s.replace(soft_unicode(old), soft_unicode(new), count)
+
+
+def do_upper(s):
+    """Convert a value to uppercase."""
+    return soft_unicode(s).upper()
+
+
+def do_lower(s):
+    """Convert a value to lowercase."""
+    return soft_unicode(s).lower()
+
+
+@evalcontextfilter
+def do_xmlattr(_eval_ctx, d, autospace=True):
+    """Create an SGML/XML attribute string based on the items in a dict.
+    All values that are neither `none` nor `undefined` are automatically
+    escaped:
+
+    .. sourcecode:: html+jinja
+
+        <ul{{ {'class': 'my_list', 'missing': none,
+                'id': 'list-%d'|format(variable)}|xmlattr }}>
+        ...
+        </ul>
+
+    Results in something like this:
+
+    .. sourcecode:: html
+
+        <ul class="my_list" id="list-42">
+        ...
+        </ul>
+
+    As you can see it automatically prepends a space in front of the item
+    if the filter returned something unless the second parameter is false.
+    """
+    rv = u' '.join(
+        u'%s="%s"' % (escape(key), escape(value))
+        for key, value in d.iteritems()
+        if value is not None and not isinstance(value, Undefined)
+    )
+    if autospace and rv:
+        rv = u' ' + rv
+    if _eval_ctx.autoescape:
+        rv = Markup(rv)
+    return rv
+
+
+def do_capitalize(s):
+    """Capitalize a value. The first character will be uppercase, all others
+    lowercase.
+    """
+    return soft_unicode(s).capitalize()
+
+
+def do_title(s):
+    """Return a titlecased version of the value. I.e. words will start with
+    uppercase letters, all remaining characters are lowercase.
+    """
+    return soft_unicode(s).title()
+
+
+def do_dictsort(value, case_sensitive=False, by='key'):
+    """Sort a dict and yield (key, value) pairs. Because python dicts are
+    unsorted you may want to use this function to order them by either
+    key or value:
+
+    .. sourcecode:: jinja
+
+        {% for item in mydict|dictsort %}
+            sort the dict by key, case insensitive
+
+        {% for item in mydict|dicsort(true) %}
+            sort the dict by key, case sensitive
+
+        {% for item in mydict|dictsort(false, 'value') %}
+            sort the dict by key, case insensitive, sorted
+            normally and ordered by value.
+    """
+    if by == 'key':
+        pos = 0
+    elif by == 'value':
+        pos = 1
+    else:
+        raise FilterArgumentError('You can only sort by either '
+                                  '"key" or "value"')
+    def sort_func(item):
+        value = item[pos]
+        if isinstance(value, basestring) and not case_sensitive:
+            value = value.lower()
+        return value
+
+    return sorted(value.items(), key=sort_func)
+
+
+def do_sort(value, reverse=False, case_sensitive=False):
+    """Sort an iterable.  Per default it sorts ascending, if you pass it
+    true as first argument it will reverse the sorting.
+
+    If the iterable is made of strings the third parameter can be used to
+    control the case sensitiveness of the comparison which is disabled by
+    default.
+
+    .. sourcecode:: jinja
+
+        {% for item in iterable|sort %}
+            ...
+        {% endfor %}
+    """
+    if not case_sensitive:
+        def sort_func(item):
+            if isinstance(item, basestring):
+                item = item.lower()
+            return item
+    else:
+        sort_func = None
+    return sorted(value, key=sort_func, reverse=reverse)
+
+
+def do_default(value, default_value=u'', boolean=False):
+    """If the value is undefined it will return the passed default value,
+    otherwise the value of the variable:
+
+    .. sourcecode:: jinja
+
+        {{ my_variable|default('my_variable is not defined') }}
+
+    This will output the value of ``my_variable`` if the variable was
+    defined, otherwise ``'my_variable is not defined'``. If you want
+    to use default with variables that evaluate to false you have to
+    set the second parameter to `true`:
+
+    .. sourcecode:: jinja
+
+        {{ ''|default('the string was empty', true) }}
+    """
+    if (boolean and not value) or isinstance(value, Undefined):
+        return default_value
+    return value
+
+
+@evalcontextfilter
+def do_join(eval_ctx, value, d=u''):
+    """Return a string which is the concatenation of the strings in the
+    sequence. The separator between elements is an empty string per
+    default, you can define it with the optional parameter:
+
+    .. sourcecode:: jinja
+
+        {{ [1, 2, 3]|join('|') }}
+            -> 1|2|3
+
+        {{ [1, 2, 3]|join }}
+            -> 123
+    """
+    # no automatic escaping?  joining is a lot eaiser then
+    if not eval_ctx.autoescape:
+        return unicode(d).join(imap(unicode, value))
+
+    # if the delimiter doesn't have an html representation we check
+    # if any of the items has.  If yes we do a coercion to Markup
+    if not hasattr(d, '__html__'):
+        value = list(value)
+        do_escape = False
+        for idx, item in enumerate(value):
+            if hasattr(item, '__html__'):
+                do_escape = True
+            else:
+                value[idx] = unicode(item)
+        if do_escape:
+            d = escape(d)
+        else:
+            d = unicode(d)
+        return d.join(value)
+
+    # no html involved, to normal joining
+    return soft_unicode(d).join(imap(soft_unicode, value))
+
+
+def do_center(value, width=80):
+    """Centers the value in a field of a given width."""
+    return unicode(value).center(width)
+
+
+@environmentfilter
+def do_first(environment, seq):
+    """Return the first item of a sequence."""
+    try:
+        return iter(seq).next()
+    except StopIteration:
+        return environment.undefined('No first item, sequence was empty.')
+
+
+@environmentfilter
+def do_last(environment, seq):
+    """Return the last item of a sequence."""
+    try:
+        return iter(reversed(seq)).next()
+    except StopIteration:
+        return environment.undefined('No last item, sequence was empty.')
+
+
+@environmentfilter
+def do_random(environment, seq):
+    """Return a random item from the sequence."""
+    try:
+        return choice(seq)
+    except IndexError:
+        return environment.undefined('No random item, sequence was empty.')
+
+
+def do_filesizeformat(value, binary=False):
+    """Format the value like a 'human-readable' file size (i.e. 13 KB,
+    4.1 MB, 102 bytes, etc).  Per default decimal prefixes are used (mega,
+    giga, etc.), if the second parameter is set to `True` the binary
+    prefixes are used (mebi, gibi).
+    """
+    bytes = float(value)
+    base = binary and 1024 or 1000
+    middle = binary and 'i' or ''
+    if bytes < base:
+        return "%d Byte%s" % (bytes, bytes != 1 and 's' or '')
+    elif bytes < base * base:
+        return "%.1f K%sB" % (bytes / base, middle)
+    elif bytes < base * base * base:
+        return "%.1f M%sB" % (bytes / (base * base), middle)
+    return "%.1f G%sB" % (bytes / (base * base * base), middle)
+
+
+def do_pprint(value, verbose=False):
+    """Pretty print a variable. Useful for debugging.
+
+    With Jinja 1.2 onwards you can pass it a parameter.  If this parameter
+    is truthy the output will be more verbose (this requires `pretty`)
+    """
+    return pformat(value, verbose=verbose)
+
+
+@evalcontextfilter
+def do_urlize(eval_ctx, value, trim_url_limit=None, nofollow=False):
+    """Converts URLs in plain text into clickable links.
+
+    If you pass the filter an additional integer it will shorten the urls
+    to that number. Also a third argument exists that makes the urls
+    "nofollow":
+
+    .. sourcecode:: jinja
+
+        {{ mytext|urlize(40, true) }}
+            links are shortened to 40 chars and defined with rel="nofollow"
+    """
+    rv = urlize(value, trim_url_limit, nofollow)
+    if eval_ctx.autoescape:
+        rv = Markup(rv)
+    return rv
+
+
+def do_indent(s, width=4, indentfirst=False):
+    """Return a copy of the passed string, each line indented by
+    4 spaces. The first line is not indented. If you want to
+    change the number of spaces or indent the first line too
+    you can pass additional parameters to the filter:
+
+    .. sourcecode:: jinja
+
+        {{ mytext|indent(2, true) }}
+            indent by two spaces and indent the first line too.
+    """
+    indention = u' ' * width
+    rv = (u'\n' + indention).join(s.splitlines())
+    if indentfirst:
+        rv = indention + rv
+    return rv
+
+
+def do_truncate(s, length=255, killwords=False, end='...'):
+    """Return a truncated copy of the string. The length is specified
+    with the first parameter which defaults to ``255``. If the second
+    parameter is ``true`` the filter will cut the text at length. Otherwise
+    it will try to save the last word. If the text was in fact
+    truncated it will append an ellipsis sign (``"..."``). If you want a
+    different ellipsis sign than ``"..."`` you can specify it using the
+    third parameter.
+
+    .. sourcecode jinja::
+
+        {{ mytext|truncate(300, false, '&raquo;') }}
+            truncate mytext to 300 chars, don't split up words, use a
+            right pointing double arrow as ellipsis sign.
+    """
+    if len(s) <= length:
+        return s
+    elif killwords:
+        return s[:length] + end
+    words = s.split(' ')
+    result = []
+    m = 0
+    for word in words:
+        m += len(word) + 1
+        if m > length:
+            break
+        result.append(word)
+    result.append(end)
+    return u' '.join(result)
+
+
+def do_wordwrap(s, width=79, break_long_words=True):
+    """
+    Return a copy of the string passed to the filter wrapped after
+    ``79`` characters.  You can override this default using the first
+    parameter.  If you set the second parameter to `false` Jinja will not
+    split words apart if they are longer than `width`.
+    """
+    import textwrap
+    return u'\n'.join(textwrap.wrap(s, width=width, expand_tabs=False,
+                                   replace_whitespace=False,
+                                   break_long_words=break_long_words))
+
+
+def do_wordcount(s):
+    """Count the words in that string."""
+    return len(_word_re.findall(s))
+
+
+def do_int(value, default=0):
+    """Convert the value into an integer. If the
+    conversion doesn't work it will return ``0``. You can
+    override this default using the first parameter.
+    """
+    try:
+        return int(value)
+    except (TypeError, ValueError):
+        # this quirk is necessary so that "42.23"|int gives 42.
+        try:
+            return int(float(value))
+        except (TypeError, ValueError):
+            return default
+
+
+def do_float(value, default=0.0):
+    """Convert the value into a floating point number. If the
+    conversion doesn't work it will return ``0.0``. You can
+    override this default using the first parameter.
+    """
+    try:
+        return float(value)
+    except (TypeError, ValueError):
+        return default
+
+
+def do_format(value, *args, **kwargs):
+    """
+    Apply python string formatting on an object:
+
+    .. sourcecode:: jinja
+
+        {{ "%s - %s"|format("Hello?", "Foo!") }}
+            -> Hello? - Foo!
+    """
+    if args and kwargs:
+        raise FilterArgumentError('can\'t handle positional and keyword '
+                                  'arguments at the same time')
+    return soft_unicode(value) % (kwargs or args)
+
+
+def do_trim(value):
+    """Strip leading and trailing whitespace."""
+    return soft_unicode(value).strip()
+
+
+def do_striptags(value):
+    """Strip SGML/XML tags and replace adjacent whitespace by one space.
+    """
+    if hasattr(value, '__html__'):
+        value = value.__html__()
+    return Markup(unicode(value)).striptags()
+
+
+def do_slice(value, slices, fill_with=None):
+    """Slice an iterator and return a list of lists containing
+    those items. Useful if you want to create a div containing
+    three ul tags that represent columns:
+
+    .. sourcecode:: html+jinja
+
+        <div class="columwrapper">
+          {%- for column in items|slice(3) %}
+            <ul class="column-{{ loop.index }}">
+            {%- for item in column %}
+              <li>{{ item }}</li>
+            {%- endfor %}
+            </ul>
+          {%- endfor %}
+        </div>
+
+    If you pass it a second argument it's used to fill missing
+    values on the last iteration.
+    """
+    seq = list(value)
+    length = len(seq)
+    items_per_slice = length // slices
+    slices_with_extra = length % slices
+    offset = 0
+    for slice_number in xrange(slices):
+        start = offset + slice_number * items_per_slice
+        if slice_number < slices_with_extra:
+            offset += 1
+        end = offset + (slice_number + 1) * items_per_slice
+        tmp = seq[start:end]
+        if fill_with is not None and slice_number >= slices_with_extra:
+            tmp.append(fill_with)
+        yield tmp
+
+
+def do_batch(value, linecount, fill_with=None):
+    """
+    A filter that batches items. It works pretty much like `slice`
+    just the other way round. It returns a list of lists with the
+    given number of items. If you provide a second parameter this
+    is used to fill missing items. See this example:
+
+    .. sourcecode:: html+jinja
+
+        <table>
+        {%- for row in items|batch(3, '&nbsp;') %}
+          <tr>
+          {%- for column in row %}
+            <td>{{ column }}</td>
+          {%- endfor %}
+          </tr>
+        {%- endfor %}
+        </table>
+    """
+    result = []
+    tmp = []
+    for item in value:
+        if len(tmp) == linecount:
+            yield tmp
+            tmp = []
+        tmp.append(item)
+    if tmp:
+        if fill_with is not None and len(tmp) < linecount:
+            tmp += [fill_with] * (linecount - len(tmp))
+        yield tmp
+
+
+def do_round(value, precision=0, method='common'):
+    """Round the number to a given precision. The first
+    parameter specifies the precision (default is ``0``), the
+    second the rounding method:
+
+    - ``'common'`` rounds either up or down
+    - ``'ceil'`` always rounds up
+    - ``'floor'`` always rounds down
+
+    If you don't specify a method ``'common'`` is used.
+
+    .. sourcecode:: jinja
+
+        {{ 42.55|round }}
+            -> 43.0
+        {{ 42.55|round(1, 'floor') }}
+            -> 42.5
+
+    Note that even if rounded to 0 precision, a float is returned.  If
+    you need a real integer, pipe it through `int`:
+
+    .. sourcecode:: jinja
+
+        {{ 42.55|round|int }}
+            -> 43
+    """
+    if not method in ('common', 'ceil', 'floor'):
+        raise FilterArgumentError('method must be common, ceil or floor')
+    if method == 'common':
+        return round(value, precision)
+    func = getattr(math, method)
+    return func(value * (10 ** precision)) / (10 ** precision)
+
+
+@environmentfilter
+def do_groupby(environment, value, attribute):
+    """Group a sequence of objects by a common attribute.
+
+    If you for example have a list of dicts or objects that represent persons
+    with `gender`, `first_name` and `last_name` attributes and you want to
+    group all users by genders you can do something like the following
+    snippet:
+
+    .. sourcecode:: html+jinja
+
+        <ul>
+        {% for group in persons|groupby('gender') %}
+            <li>{{ group.grouper }}<ul>
+            {% for person in group.list %}
+                <li>{{ person.first_name }} {{ person.last_name }}</li>
+            {% endfor %}</ul></li>
+        {% endfor %}
+        </ul>
+
+    Additionally it's possible to use tuple unpacking for the grouper and
+    list:
+
+    .. sourcecode:: html+jinja
+
+        <ul>
+        {% for grouper, list in persons|groupby('gender') %}
+            ...
+        {% endfor %}
+        </ul>
+
+    As you can see the item we're grouping by is stored in the `grouper`
+    attribute and the `list` contains all the objects that have this grouper
+    in common.
+    """
+    expr = lambda x: environment.getitem(x, attribute)
+    return sorted(map(_GroupTuple, groupby(sorted(value, key=expr), expr)))
+
+
+class _GroupTuple(tuple):
+    __slots__ = ()
+    grouper = property(itemgetter(0))
+    list = property(itemgetter(1))
+
+    def __new__(cls, (key, value)):
+        return tuple.__new__(cls, (key, list(value)))
+
+
+def do_list(value):
+    """Convert the value into a list.  If it was a string the returned list
+    will be a list of characters.
+    """
+    return list(value)
+
+
+def do_mark_safe(value):
+    """Mark the value as safe which means that in an environment with automatic
+    escaping enabled this variable will not be escaped.
+    """
+    return Markup(value)
+
+
+def do_mark_unsafe(value):
+    """Mark a value as unsafe.  This is the reverse operation for :func:`safe`."""
+    return unicode(value)
+
+
+def do_reverse(value):
+    """Reverse the object or return an iterator the iterates over it the other
+    way round.
+    """
+    if isinstance(value, basestring):
+        return value[::-1]
+    try:
+        return reversed(value)
+    except TypeError:
+        try:
+            rv = list(value)
+            rv.reverse()
+            return rv
+        except TypeError:
+            raise FilterArgumentError('argument must be iterable')
+
+
+@environmentfilter
+def do_attr(environment, obj, name):
+    """Get an attribute of an object.  ``foo|attr("bar")`` works like
+    ``foo["bar"]`` just that always an attribute is returned and items are not
+    looked up.
+
+    See :ref:`Notes on subscriptions <notes-on-subscriptions>` for more details.
+    """
+    try:
+        name = str(name)
+    except UnicodeError:
+        pass
+    else:
+        try:
+            value = getattr(obj, name)
+        except AttributeError:
+            pass
+        else:
+            if environment.sandboxed and not \
+               environment.is_safe_attribute(obj, name, value):
+                return environment.unsafe_undefined(obj, name)
+            return value
+    return environment.undefined(obj=obj, name=name)
+
+
+FILTERS = {
+    'attr':                 do_attr,
+    'replace':              do_replace,
+    'upper':                do_upper,
+    'lower':                do_lower,
+    'escape':               escape,
+    'e':                    escape,
+    'forceescape':          do_forceescape,
+    'capitalize':           do_capitalize,
+    'title':                do_title,
+    'default':              do_default,
+    'd':                    do_default,
+    'join':                 do_join,
+    'count':                len,
+    'dictsort':             do_dictsort,
+    'sort':                 do_sort,
+    'length':               len,
+    'reverse':              do_reverse,
+    'center':               do_center,
+    'indent':               do_indent,
+    'title':                do_title,
+    'capitalize':           do_capitalize,
+    'first':                do_first,
+    'last':                 do_last,
+    'random':               do_random,
+    'filesizeformat':       do_filesizeformat,
+    'pprint':               do_pprint,
+    'truncate':             do_truncate,
+    'wordwrap':             do_wordwrap,
+    'wordcount':            do_wordcount,
+    'int':                  do_int,
+    'float':                do_float,
+    'string':               soft_unicode,
+    'list':                 do_list,
+    'urlize':               do_urlize,
+    'format':               do_format,
+    'trim':                 do_trim,
+    'striptags':            do_striptags,
+    'slice':                do_slice,
+    'batch':                do_batch,
+    'sum':                  sum,
+    'abs':                  abs,
+    'round':                do_round,
+    'groupby':              do_groupby,
+    'safe':                 do_mark_safe,
+    'xmlattr':              do_xmlattr
+}

http://git-wip-us.apache.org/repos/asf/ambari/blob/658360a5/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/lexer.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/lexer.py b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/lexer.py
new file mode 100644
index 0000000..0c385c1
--- /dev/null
+++ b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/lexer.py
@@ -0,0 +1,681 @@
+# -*- coding: utf-8 -*-
+"""
+    ambari_jinja2.lexer
+    ~~~~~~~~~~~~
+
+    This module implements a Jinja / Python combination lexer. The
+    `Lexer` class provided by this module is used to do some preprocessing
+    for Jinja.
+
+    On the one hand it filters out invalid operators like the bitshift
+    operators we don't allow in templates. On the other hand it separates
+    template code and python code in expressions.
+
+    :copyright: (c) 2010 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+import re
+from operator import itemgetter
+from collections import deque
+from ambari_jinja2.exceptions import TemplateSyntaxError
+from ambari_jinja2.utils import LRUCache, next
+
+
+# cache for the lexers. Exists in order to be able to have multiple
+# environments with the same lexer
+_lexer_cache = LRUCache(50)
+
+# static regular expressions
+whitespace_re = re.compile(r'\s+', re.U)
+string_re = re.compile(r"('([^'\\]*(?:\\.[^'\\]*)*)'"
+                       r'|"([^"\\]*(?:\\.[^"\\]*)*)")', re.S)
+integer_re = re.compile(r'\d+')
+
+# we use the unicode identifier rule if this python version is able
+# to handle unicode identifiers, otherwise the standard ASCII one.
+try:
+    compile('föö', '<unknown>', 'eval')
+except SyntaxError:
+    name_re = re.compile(r'\b[a-zA-Z_][a-zA-Z0-9_]*\b')
+else:
+    from ambari_jinja2 import _stringdefs
+    name_re = re.compile(r'[%s][%s]*' % (_stringdefs.xid_start,
+                                         _stringdefs.xid_continue))
+
+float_re = re.compile(r'(?<!\.)\d+\.\d+')
+newline_re = re.compile(r'(\r\n|\r|\n)')
+
+# internal the tokens and keep references to them
+TOKEN_ADD = intern('add')
+TOKEN_ASSIGN = intern('assign')
+TOKEN_COLON = intern('colon')
+TOKEN_COMMA = intern('comma')
+TOKEN_DIV = intern('div')
+TOKEN_DOT = intern('dot')
+TOKEN_EQ = intern('eq')
+TOKEN_FLOORDIV = intern('floordiv')
+TOKEN_GT = intern('gt')
+TOKEN_GTEQ = intern('gteq')
+TOKEN_LBRACE = intern('lbrace')
+TOKEN_LBRACKET = intern('lbracket')
+TOKEN_LPAREN = intern('lparen')
+TOKEN_LT = intern('lt')
+TOKEN_LTEQ = intern('lteq')
+TOKEN_MOD = intern('mod')
+TOKEN_MUL = intern('mul')
+TOKEN_NE = intern('ne')
+TOKEN_PIPE = intern('pipe')
+TOKEN_POW = intern('pow')
+TOKEN_RBRACE = intern('rbrace')
+TOKEN_RBRACKET = intern('rbracket')
+TOKEN_RPAREN = intern('rparen')
+TOKEN_SEMICOLON = intern('semicolon')
+TOKEN_SUB = intern('sub')
+TOKEN_TILDE = intern('tilde')
+TOKEN_WHITESPACE = intern('whitespace')
+TOKEN_FLOAT = intern('float')
+TOKEN_INTEGER = intern('integer')
+TOKEN_NAME = intern('name')
+TOKEN_STRING = intern('string')
+TOKEN_OPERATOR = intern('operator')
+TOKEN_BLOCK_BEGIN = intern('block_begin')
+TOKEN_BLOCK_END = intern('block_end')
+TOKEN_VARIABLE_BEGIN = intern('variable_begin')
+TOKEN_VARIABLE_END = intern('variable_end')
+TOKEN_RAW_BEGIN = intern('raw_begin')
+TOKEN_RAW_END = intern('raw_end')
+TOKEN_COMMENT_BEGIN = intern('comment_begin')
+TOKEN_COMMENT_END = intern('comment_end')
+TOKEN_COMMENT = intern('comment')
+TOKEN_LINESTATEMENT_BEGIN = intern('linestatement_begin')
+TOKEN_LINESTATEMENT_END = intern('linestatement_end')
+TOKEN_LINECOMMENT_BEGIN = intern('linecomment_begin')
+TOKEN_LINECOMMENT_END = intern('linecomment_end')
+TOKEN_LINECOMMENT = intern('linecomment')
+TOKEN_DATA = intern('data')
+TOKEN_INITIAL = intern('initial')
+TOKEN_EOF = intern('eof')
+
+# bind operators to token types
+operators = {
+    '+':            TOKEN_ADD,
+    '-':            TOKEN_SUB,
+    '/':            TOKEN_DIV,
+    '//':           TOKEN_FLOORDIV,
+    '*':            TOKEN_MUL,
+    '%':            TOKEN_MOD,
+    '**':           TOKEN_POW,
+    '~':            TOKEN_TILDE,
+    '[':            TOKEN_LBRACKET,
+    ']':            TOKEN_RBRACKET,
+    '(':            TOKEN_LPAREN,
+    ')':            TOKEN_RPAREN,
+    '{':            TOKEN_LBRACE,
+    '}':            TOKEN_RBRACE,
+    '==':           TOKEN_EQ,
+    '!=':           TOKEN_NE,
+    '>':            TOKEN_GT,
+    '>=':           TOKEN_GTEQ,
+    '<':            TOKEN_LT,
+    '<=':           TOKEN_LTEQ,
+    '=':            TOKEN_ASSIGN,
+    '.':            TOKEN_DOT,
+    ':':            TOKEN_COLON,
+    '|':            TOKEN_PIPE,
+    ',':            TOKEN_COMMA,
+    ';':            TOKEN_SEMICOLON
+}
+
+reverse_operators = dict([(v, k) for k, v in operators.iteritems()])
+assert len(operators) == len(reverse_operators), 'operators dropped'
+operator_re = re.compile('(%s)' % '|'.join(re.escape(x) for x in
+                         sorted(operators, key=lambda x: -len(x))))
+
+ignored_tokens = frozenset([TOKEN_COMMENT_BEGIN, TOKEN_COMMENT,
+                            TOKEN_COMMENT_END, TOKEN_WHITESPACE,
+                            TOKEN_WHITESPACE, TOKEN_LINECOMMENT_BEGIN,
+                            TOKEN_LINECOMMENT_END, TOKEN_LINECOMMENT])
+ignore_if_empty = frozenset([TOKEN_WHITESPACE, TOKEN_DATA,
+                             TOKEN_COMMENT, TOKEN_LINECOMMENT])
+
+
+def _describe_token_type(token_type):
+    if token_type in reverse_operators:
+        return reverse_operators[token_type]
+    return {
+        TOKEN_COMMENT_BEGIN:        'begin of comment',
+        TOKEN_COMMENT_END:          'end of comment',
+        TOKEN_COMMENT:              'comment',
+        TOKEN_LINECOMMENT:          'comment',
+        TOKEN_BLOCK_BEGIN:          'begin of statement block',
+        TOKEN_BLOCK_END:            'end of statement block',
+        TOKEN_VARIABLE_BEGIN:       'begin of print statement',
+        TOKEN_VARIABLE_END:         'end of print statement',
+        TOKEN_LINESTATEMENT_BEGIN:  'begin of line statement',
+        TOKEN_LINESTATEMENT_END:    'end of line statement',
+        TOKEN_DATA:                 'template data / text',
+        TOKEN_EOF:                  'end of template'
+    }.get(token_type, token_type)
+
+
+def describe_token(token):
+    """Returns a description of the token."""
+    if token.type == 'name':
+        return token.value
+    return _describe_token_type(token.type)
+
+
+def describe_token_expr(expr):
+    """Like `describe_token` but for token expressions."""
+    if ':' in expr:
+        type, value = expr.split(':', 1)
+        if type == 'name':
+            return value
+    else:
+        type = expr
+    return _describe_token_type(type)
+
+
+def count_newlines(value):
+    """Count the number of newline characters in the string.  This is
+    useful for extensions that filter a stream.
+    """
+    return len(newline_re.findall(value))
+
+
+def compile_rules(environment):
+    """Compiles all the rules from the environment into a list of rules."""
+    e = re.escape
+    rules = [
+        (len(environment.comment_start_string), 'comment',
+         e(environment.comment_start_string)),
+        (len(environment.block_start_string), 'block',
+         e(environment.block_start_string)),
+        (len(environment.variable_start_string), 'variable',
+         e(environment.variable_start_string))
+    ]
+
+    if environment.line_statement_prefix is not None:
+        rules.append((len(environment.line_statement_prefix), 'linestatement',
+                      r'^\s*' + e(environment.line_statement_prefix)))
+    if environment.line_comment_prefix is not None:
+        rules.append((len(environment.line_comment_prefix), 'linecomment',
+                      r'(?:^|(?<=\S))[^\S\r\n]*' +
+                      e(environment.line_comment_prefix)))
+
+    return [x[1:] for x in sorted(rules, reverse=True)]
+
+
+class Failure(object):
+    """Class that raises a `TemplateSyntaxError` if called.
+    Used by the `Lexer` to specify known errors.
+    """
+
+    def __init__(self, message, cls=TemplateSyntaxError):
+        self.message = message
+        self.error_class = cls
+
+    def __call__(self, lineno, filename):
+        raise self.error_class(self.message, lineno, filename)
+
+
+class Token(tuple):
+    """Token class."""
+    __slots__ = ()
+    lineno, type, value = (property(itemgetter(x)) for x in range(3))
+
+    def __new__(cls, lineno, type, value):
+        return tuple.__new__(cls, (lineno, intern(str(type)), value))
+
+    def __str__(self):
+        if self.type in reverse_operators:
+            return reverse_operators[self.type]
+        elif self.type == 'name':
+            return self.value
+        return self.type
+
+    def test(self, expr):
+        """Test a token against a token expression.  This can either be a
+        token type or ``'token_type:token_value'``.  This can only test
+        against string values and types.
+        """
+        # here we do a regular string equality check as test_any is usually
+        # passed an iterable of not interned strings.
+        if self.type == expr:
+            return True
+        elif ':' in expr:
+            return expr.split(':', 1) == [self.type, self.value]
+        return False
+
+    def test_any(self, *iterable):
+        """Test against multiple token expressions."""
+        for expr in iterable:
+            if self.test(expr):
+                return True
+        return False
+
+    def __repr__(self):
+        return 'Token(%r, %r, %r)' % (
+            self.lineno,
+            self.type,
+            self.value
+        )
+
+
+class TokenStreamIterator(object):
+    """The iterator for tokenstreams.  Iterate over the stream
+    until the eof token is reached.
+    """
+
+    def __init__(self, stream):
+        self.stream = stream
+
+    def __iter__(self):
+        return self
+
+    def next(self):
+        token = self.stream.current
+        if token.type is TOKEN_EOF:
+            self.stream.close()
+            raise StopIteration()
+        next(self.stream)
+        return token
+
+
+class TokenStream(object):
+    """A token stream is an iterable that yields :class:`Token`\s.  The
+    parser however does not iterate over it but calls :meth:`next` to go
+    one token ahead.  The current active token is stored as :attr:`current`.
+    """
+
+    def __init__(self, generator, name, filename):
+        self._next = iter(generator).next
+        self._pushed = deque()
+        self.name = name
+        self.filename = filename
+        self.closed = False
+        self.current = Token(1, TOKEN_INITIAL, '')
+        next(self)
+
+    def __iter__(self):
+        return TokenStreamIterator(self)
+
+    def __nonzero__(self):
+        return bool(self._pushed) or self.current.type is not TOKEN_EOF
+
+    eos = property(lambda x: not x, doc="Are we at the end of the stream?")
+
+    def push(self, token):
+        """Push a token back to the stream."""
+        self._pushed.append(token)
+
+    def look(self):
+        """Look at the next token."""
+        old_token = next(self)
+        result = self.current
+        self.push(result)
+        self.current = old_token
+        return result
+
+    def skip(self, n=1):
+        """Got n tokens ahead."""
+        for x in xrange(n):
+            next(self)
+
+    def next_if(self, expr):
+        """Perform the token test and return the token if it matched.
+        Otherwise the return value is `None`.
+        """
+        if self.current.test(expr):
+            return next(self)
+
+    def skip_if(self, expr):
+        """Like :meth:`next_if` but only returns `True` or `False`."""
+        return self.next_if(expr) is not None
+
+    def next(self):
+        """Go one token ahead and return the old one"""
+        rv = self.current
+        if self._pushed:
+            self.current = self._pushed.popleft()
+        elif self.current.type is not TOKEN_EOF:
+            try:
+                self.current = self._next()
+            except StopIteration:
+                self.close()
+        return rv
+
+    def close(self):
+        """Close the stream."""
+        self.current = Token(self.current.lineno, TOKEN_EOF, '')
+        self._next = None
+        self.closed = True
+
+    def expect(self, expr):
+        """Expect a given token type and return it.  This accepts the same
+        argument as :meth:`ambari_jinja2.lexer.Token.test`.
+        """
+        if not self.current.test(expr):
+            expr = describe_token_expr(expr)
+            if self.current.type is TOKEN_EOF:
+                raise TemplateSyntaxError('unexpected end of template, '
+                                          'expected %r.' % expr,
+                                          self.current.lineno,
+                                          self.name, self.filename)
+            raise TemplateSyntaxError("expected token %r, got %r" %
+                                      (expr, describe_token(self.current)),
+                                      self.current.lineno,
+                                      self.name, self.filename)
+        try:
+            return self.current
+        finally:
+            next(self)
+
+
+def get_lexer(environment):
+    """Return a lexer which is probably cached."""
+    key = (environment.block_start_string,
+           environment.block_end_string,
+           environment.variable_start_string,
+           environment.variable_end_string,
+           environment.comment_start_string,
+           environment.comment_end_string,
+           environment.line_statement_prefix,
+           environment.line_comment_prefix,
+           environment.trim_blocks,
+           environment.newline_sequence)
+    lexer = _lexer_cache.get(key)
+    if lexer is None:
+        lexer = Lexer(environment)
+        _lexer_cache[key] = lexer
+    return lexer
+
+
+class Lexer(object):
+    """Class that implements a lexer for a given environment. Automatically
+    created by the environment class, usually you don't have to do that.
+
+    Note that the lexer is not automatically bound to an environment.
+    Multiple environments can share the same lexer.
+    """
+
+    def __init__(self, environment):
+        # shortcuts
+        c = lambda x: re.compile(x, re.M | re.S)
+        e = re.escape
+
+        # lexing rules for tags
+        tag_rules = [
+            (whitespace_re, TOKEN_WHITESPACE, None),
+            (float_re, TOKEN_FLOAT, None),
+            (integer_re, TOKEN_INTEGER, None),
+            (name_re, TOKEN_NAME, None),
+            (string_re, TOKEN_STRING, None),
+            (operator_re, TOKEN_OPERATOR, None)
+        ]
+
+        # assamble the root lexing rule. because "|" is ungreedy
+        # we have to sort by length so that the lexer continues working
+        # as expected when we have parsing rules like <% for block and
+        # <%= for variables. (if someone wants asp like syntax)
+        # variables are just part of the rules if variable processing
+        # is required.
+        root_tag_rules = compile_rules(environment)
+
+        # block suffix if trimming is enabled
+        block_suffix_re = environment.trim_blocks and '\\n?' or ''
+
+        self.newline_sequence = environment.newline_sequence
+
+        # global lexing rules
+        self.rules = {
+            'root': [
+                # directives
+                (c('(.*?)(?:%s)' % '|'.join(
+                    [r'(?P<raw_begin>(?:\s*%s\-|%s)\s*raw\s*(?:\-%s\s*|%s))' % (
+                        e(environment.block_start_string),
+                        e(environment.block_start_string),
+                        e(environment.block_end_string),
+                        e(environment.block_end_string)
+                    )] + [
+                        r'(?P<%s_begin>\s*%s\-|%s)' % (n, r, r)
+                        for n, r in root_tag_rules
+                    ])), (TOKEN_DATA, '#bygroup'), '#bygroup'),
+                # data
+                (c('.+'), TOKEN_DATA, None)
+            ],
+            # comments
+            TOKEN_COMMENT_BEGIN: [
+                (c(r'(.*?)((?:\-%s\s*|%s)%s)' % (
+                    e(environment.comment_end_string),
+                    e(environment.comment_end_string),
+                    block_suffix_re
+                )), (TOKEN_COMMENT, TOKEN_COMMENT_END), '#pop'),
+                (c('(.)'), (Failure('Missing end of comment tag'),), None)
+            ],
+            # blocks
+            TOKEN_BLOCK_BEGIN: [
+                (c('(?:\-%s\s*|%s)%s' % (
+                    e(environment.block_end_string),
+                    e(environment.block_end_string),
+                    block_suffix_re
+                )), TOKEN_BLOCK_END, '#pop'),
+            ] + tag_rules,
+            # variables
+            TOKEN_VARIABLE_BEGIN: [
+                (c('\-%s\s*|%s' % (
+                    e(environment.variable_end_string),
+                    e(environment.variable_end_string)
+                )), TOKEN_VARIABLE_END, '#pop')
+            ] + tag_rules,
+            # raw block
+            TOKEN_RAW_BEGIN: [
+                (c('(.*?)((?:\s*%s\-|%s)\s*endraw\s*(?:\-%s\s*|%s%s))' % (
+                    e(environment.block_start_string),
+                    e(environment.block_start_string),
+                    e(environment.block_end_string),
+                    e(environment.block_end_string),
+                    block_suffix_re
+                )), (TOKEN_DATA, TOKEN_RAW_END), '#pop'),
+                (c('(.)'), (Failure('Missing end of raw directive'),), None)
+            ],
+            # line statements
+            TOKEN_LINESTATEMENT_BEGIN: [
+                (c(r'\s*(\n|$)'), TOKEN_LINESTATEMENT_END, '#pop')
+            ] + tag_rules,
+            # line comments
+            TOKEN_LINECOMMENT_BEGIN: [
+                (c(r'(.*?)()(?=\n|$)'), (TOKEN_LINECOMMENT,
+                 TOKEN_LINECOMMENT_END), '#pop')
+            ]
+        }
+
+    def _normalize_newlines(self, value):
+        """Called for strings and template data to normlize it to unicode."""
+        return newline_re.sub(self.newline_sequence, value)
+
+    def tokenize(self, source, name=None, filename=None, state=None):
+        """Calls tokeniter + tokenize and wraps it in a token stream.
+        """
+        stream = self.tokeniter(source, name, filename, state)
+        return TokenStream(self.wrap(stream, name, filename), name, filename)
+
+    def wrap(self, stream, name=None, filename=None):
+        """This is called with the stream as returned by `tokenize` and wraps
+        every token in a :class:`Token` and converts the value.
+        """
+        for lineno, token, value in stream:
+            if token in ignored_tokens:
+                continue
+            elif token == 'linestatement_begin':
+                token = 'block_begin'
+            elif token == 'linestatement_end':
+                token = 'block_end'
+            # we are not interested in those tokens in the parser
+            elif token in ('raw_begin', 'raw_end'):
+                continue
+            elif token == 'data':
+                value = self._normalize_newlines(value)
+            elif token == 'keyword':
+                token = value
+            elif token == 'name':
+                value = str(value)
+            elif token == 'string':
+                # try to unescape string
+                try:
+                    value = self._normalize_newlines(value[1:-1]) \
+                        .encode('ascii', 'backslashreplace') \
+                        .decode('unicode-escape')
+                except Exception, e:
+                    msg = str(e).split(':')[-1].strip()
+                    raise TemplateSyntaxError(msg, lineno, name, filename)
+                # if we can express it as bytestring (ascii only)
+                # we do that for support of semi broken APIs
+                # as datetime.datetime.strftime.  On python 3 this
+                # call becomes a noop thanks to 2to3
+                try:
+                    value = str(value)
+                except UnicodeError:
+                    pass
+            elif token == 'integer':
+                value = int(value)
+            elif token == 'float':
+                value = float(value)
+            elif token == 'operator':
+                token = operators[value]
+            yield Token(lineno, token, value)
+
+    def tokeniter(self, source, name, filename=None, state=None):
+        """This method tokenizes the text and returns the tokens in a
+        generator.  Use this method if you just want to tokenize a template.
+        """
+        source = '\n'.join(unicode(source).splitlines())
+        pos = 0
+        lineno = 1
+        stack = ['root']
+        if state is not None and state != 'root':
+            assert state in ('variable', 'block'), 'invalid state'
+            stack.append(state + '_begin')
+        else:
+            state = 'root'
+        statetokens = self.rules[stack[-1]]
+        source_length = len(source)
+
+        balancing_stack = []
+
+        while 1:
+            # tokenizer loop
+            for regex, tokens, new_state in statetokens:
+                m = regex.match(source, pos)
+                # if no match we try again with the next rule
+                if m is None:
+                    continue
+
+                # we only match blocks and variables if brances / parentheses
+                # are balanced. continue parsing with the lower rule which
+                # is the operator rule. do this only if the end tags look
+                # like operators
+                if balancing_stack and \
+                   tokens in ('variable_end', 'block_end',
+                              'linestatement_end'):
+                    continue
+
+                # tuples support more options
+                if isinstance(tokens, tuple):
+                    for idx, token in enumerate(tokens):
+                        # failure group
+                        if token.__class__ is Failure:
+                            raise token(lineno, filename)
+                        # bygroup is a bit more complex, in that case we
+                        # yield for the current token the first named
+                        # group that matched
+                        elif token == '#bygroup':
+                            for key, value in m.groupdict().iteritems():
+                                if value is not None:
+                                    yield lineno, key, value
+                                    lineno += value.count('\n')
+                                    break
+                            else:
+                                raise RuntimeError('%r wanted to resolve '
+                                                   'the token dynamically'
+                                                   ' but no group matched'
+                                                   % regex)
+                        # normal group
+                        else:
+                            data = m.group(idx + 1)
+                            if data or token not in ignore_if_empty:
+                                yield lineno, token, data
+                            lineno += data.count('\n')
+
+                # strings as token just are yielded as it.
+                else:
+                    data = m.group()
+                    # update brace/parentheses balance
+                    if tokens == 'operator':
+                        if data == '{':
+                            balancing_stack.append('}')
+                        elif data == '(':
+                            balancing_stack.append(')')
+                        elif data == '[':
+                            balancing_stack.append(']')
+                        elif data in ('}', ')', ']'):
+                            if not balancing_stack:
+                                raise TemplateSyntaxError('unexpected \'%s\'' %
+                                                          data, lineno, name,
+                                                          filename)
+                            expected_op = balancing_stack.pop()
+                            if expected_op != data:
+                                raise TemplateSyntaxError('unexpected \'%s\', '
+                                                          'expected \'%s\'' %
+                                                          (data, expected_op),
+                                                          lineno, name,
+                                                          filename)
+                    # yield items
+                    if data or tokens not in ignore_if_empty:
+                        yield lineno, tokens, data
+                    lineno += data.count('\n')
+
+                # fetch new position into new variable so that we can check
+                # if there is a internal parsing error which would result
+                # in an infinite loop
+                pos2 = m.end()
+
+                # handle state changes
+                if new_state is not None:
+                    # remove the uppermost state
+                    if new_state == '#pop':
+                        stack.pop()
+                    # resolve the new state by group checking
+                    elif new_state == '#bygroup':
+                        for key, value in m.groupdict().iteritems():
+                            if value is not None:
+                                stack.append(key)
+                                break
+                        else:
+                            raise RuntimeError('%r wanted to resolve the '
+                                               'new state dynamically but'
+                                               ' no group matched' %
+                                               regex)
+                    # direct state name given
+                    else:
+                        stack.append(new_state)
+                    statetokens = self.rules[stack[-1]]
+                # we are still at the same position and no stack change.
+                # this means a loop without break condition, avoid that and
+                # raise error
+                elif pos2 == pos:
+                    raise RuntimeError('%r yielded empty string without '
+                                       'stack change' % regex)
+                # publish new function and start again
+                pos = pos2
+                break
+            # if loop terminated without break we havn't found a single match
+            # either we are at the end of the file or we have a problem
+            else:
+                # end of text
+                if pos >= source_length:
+                    return
+                # something went wrong
+                raise TemplateSyntaxError('unexpected char %r at %d' %
+                                          (source[pos], pos), lineno,
+                                          name, filename)

http://git-wip-us.apache.org/repos/asf/ambari/blob/658360a5/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/loaders.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/loaders.py b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/loaders.py
new file mode 100644
index 0000000..b25be08
--- /dev/null
+++ b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/loaders.py
@@ -0,0 +1,449 @@
+# -*- coding: utf-8 -*-
+"""
+    ambari_jinja2.loaders
+    ~~~~~~~~~~~~~~
+
+    Jinja loader classes.
+
+    :copyright: (c) 2010 by the Jinja Team.
+    :license: BSD, see LICENSE for more details.
+"""
+import os
+import sys
+import weakref
+from types import ModuleType
+from os import path
+try:
+    from hashlib import sha1
+except ImportError:
+    from sha import new as sha1
+from ambari_jinja2.exceptions import TemplateNotFound
+from ambari_jinja2.utils import LRUCache, open_if_exists, internalcode
+
+
+def split_template_path(template):
+    """Split a path into segments and perform a sanity check.  If it detects
+    '..' in the path it will raise a `TemplateNotFound` error.
+    """
+    pieces = []
+    for piece in template.split('/'):
+        if path.sep in piece \
+           or (path.altsep and path.altsep in piece) or \
+           piece == path.pardir:
+            raise TemplateNotFound(template)
+        elif piece and piece != '.':
+            pieces.append(piece)
+    return pieces
+
+
+class BaseLoader(object):
+    """Baseclass for all loaders.  Subclass this and override `get_source` to
+    implement a custom loading mechanism.  The environment provides a
+    `get_template` method that calls the loader's `load` method to get the
+    :class:`Template` object.
+
+    A very basic example for a loader that looks up templates on the file
+    system could look like this::
+
+        from ambari_jinja2 import BaseLoader, TemplateNotFound
+        from os.path import join, exists, getmtime
+
+        class MyLoader(BaseLoader):
+
+            def __init__(self, path):
+                self.path = path
+
+            def get_source(self, environment, template):
+                path = join(self.path, template)
+                if not exists(path):
+                    raise TemplateNotFound(template)
+                mtime = getmtime(path)
+                with file(path) as f:
+                    source = f.read().decode('utf-8')
+                return source, path, lambda: mtime == getmtime(path)
+    """
+
+    #: if set to `False` it indicates that the loader cannot provide access
+    #: to the source of templates.
+    #:
+    #: .. versionadded:: 2.4
+    has_source_access = True
+
+    def get_source(self, environment, template):
+        """Get the template source, filename and reload helper for a template.
+        It's passed the environment and template name and has to return a
+        tuple in the form ``(source, filename, uptodate)`` or raise a
+        `TemplateNotFound` error if it can't locate the template.
+
+        The source part of the returned tuple must be the source of the
+        template as unicode string or a ASCII bytestring.  The filename should
+        be the name of the file on the filesystem if it was loaded from there,
+        otherwise `None`.  The filename is used by python for the tracebacks
+        if no loader extension is used.
+
+        The last item in the tuple is the `uptodate` function.  If auto
+        reloading is enabled it's always called to check if the template
+        changed.  No arguments are passed so the function must store the
+        old state somewhere (for example in a closure).  If it returns `False`
+        the template will be reloaded.
+        """
+        if not self.has_source_access:
+            raise RuntimeError('%s cannot provide access to the source' %
+                               self.__class__.__name__)
+        raise TemplateNotFound(template)
+
+    def list_templates(self):
+        """Iterates over all templates.  If the loader does not support that
+        it should raise a :exc:`TypeError` which is the default behavior.
+        """
+        raise TypeError('this loader cannot iterate over all templates')
+
+    @internalcode
+    def load(self, environment, name, globals=None):
+        """Loads a template.  This method looks up the template in the cache
+        or loads one by calling :meth:`get_source`.  Subclasses should not
+        override this method as loaders working on collections of other
+        loaders (such as :class:`PrefixLoader` or :class:`ChoiceLoader`)
+        will not call this method but `get_source` directly.
+        """
+        code = None
+        if globals is None:
+            globals = {}
+
+        # first we try to get the source for this template together
+        # with the filename and the uptodate function.
+        source, filename, uptodate = self.get_source(environment, name)
+
+        # try to load the code from the bytecode cache if there is a
+        # bytecode cache configured.
+        bcc = environment.bytecode_cache
+        if bcc is not None:
+            bucket = bcc.get_bucket(environment, name, filename, source)
+            code = bucket.code
+
+        # if we don't have code so far (not cached, no longer up to
+        # date) etc. we compile the template
+        if code is None:
+            code = environment.compile(source, name, filename)
+
+        # if the bytecode cache is available and the bucket doesn't
+        # have a code so far, we give the bucket the new code and put
+        # it back to the bytecode cache.
+        if bcc is not None and bucket.code is None:
+            bucket.code = code
+            bcc.set_bucket(bucket)
+
+        return environment.template_class.from_code(environment, code,
+                                                    globals, uptodate)
+
+
+class FileSystemLoader(BaseLoader):
+    """Loads templates from the file system.  This loader can find templates
+    in folders on the file system and is the preferred way to load them.
+
+    The loader takes the path to the templates as string, or if multiple
+    locations are wanted a list of them which is then looked up in the
+    given order:
+
+    >>> loader = FileSystemLoader('/path/to/templates')
+    >>> loader = FileSystemLoader(['/path/to/templates', '/other/path'])
+
+    Per default the template encoding is ``'utf-8'`` which can be changed
+    by setting the `encoding` parameter to something else.
+    """
+
+    def __init__(self, searchpath, encoding='utf-8'):
+        if isinstance(searchpath, basestring):
+            searchpath = [searchpath]
+        self.searchpath = list(searchpath)
+        self.encoding = encoding
+
+    def get_source(self, environment, template):
+        pieces = split_template_path(template)
+        for searchpath in self.searchpath:
+            filename = path.join(searchpath, *pieces)
+            f = open_if_exists(filename)
+            if f is None:
+                continue
+            try:
+                contents = f.read().decode(self.encoding)
+            finally:
+                f.close()
+
+            mtime = path.getmtime(filename)
+            def uptodate():
+                try:
+                    return path.getmtime(filename) == mtime
+                except OSError:
+                    return False
+            return contents, filename, uptodate
+        raise TemplateNotFound(template)
+
+    def list_templates(self):
+        found = set()
+        for searchpath in self.searchpath:
+            for dirpath, dirnames, filenames in os.walk(searchpath):
+                for filename in filenames:
+                    template = os.path.join(dirpath, filename) \
+                        [len(searchpath):].strip(os.path.sep) \
+                                          .replace(os.path.sep, '/')
+                    if template[:2] == './':
+                        template = template[2:]
+                    if template not in found:
+                        found.add(template)
+        return sorted(found)
+
+
+class PackageLoader(BaseLoader):
+    """Load templates from python eggs or packages.  It is constructed with
+    the name of the python package and the path to the templates in that
+    package::
+
+        loader = PackageLoader('mypackage', 'views')
+
+    If the package path is not given, ``'templates'`` is assumed.
+
+    Per default the template encoding is ``'utf-8'`` which can be changed
+    by setting the `encoding` parameter to something else.  Due to the nature
+    of eggs it's only possible to reload templates if the package was loaded
+    from the file system and not a zip file.
+    """
+
+    def __init__(self, package_name, package_path='templates',
+                 encoding='utf-8'):
+        from pkg_resources import DefaultProvider, ResourceManager, \
+                                  get_provider
+        provider = get_provider(package_name)
+        self.encoding = encoding
+        self.manager = ResourceManager()
+        self.filesystem_bound = isinstance(provider, DefaultProvider)
+        self.provider = provider
+        self.package_path = package_path
+
+    def get_source(self, environment, template):
+        pieces = split_template_path(template)
+        p = '/'.join((self.package_path,) + tuple(pieces))
+        if not self.provider.has_resource(p):
+            raise TemplateNotFound(template)
+
+        filename = uptodate = None
+        if self.filesystem_bound:
+            filename = self.provider.get_resource_filename(self.manager, p)
+            mtime = path.getmtime(filename)
+            def uptodate():
+                try:
+                    return path.getmtime(filename) == mtime
+                except OSError:
+                    return False
+
+        source = self.provider.get_resource_string(self.manager, p)
+        return source.decode(self.encoding), filename, uptodate
+
+    def list_templates(self):
+        path = self.package_path
+        if path[:2] == './':
+            path = path[2:]
+        elif path == '.':
+            path = ''
+        offset = len(path)
+        results = []
+        def _walk(path):
+            for filename in self.provider.resource_listdir(path):
+                fullname = path + '/' + filename
+                if self.provider.resource_isdir(fullname):
+                    for item in _walk(fullname):
+                        results.append(item)
+                else:
+                    results.append(fullname[offset:].lstrip('/'))
+        _walk(path)
+        results.sort()
+        return results
+
+
+class DictLoader(BaseLoader):
+    """Loads a template from a python dict.  It's passed a dict of unicode
+    strings bound to template names.  This loader is useful for unittesting:
+
+    >>> loader = DictLoader({'index.html': 'source here'})
+
+    Because auto reloading is rarely useful this is disabled per default.
+    """
+
+    def __init__(self, mapping):
+        self.mapping = mapping
+
+    def get_source(self, environment, template):
+        if template in self.mapping:
+            source = self.mapping[template]
+            return source, None, lambda: source != self.mapping.get(template)
+        raise TemplateNotFound(template)
+
+    def list_templates(self):
+        return sorted(self.mapping)
+
+
+class FunctionLoader(BaseLoader):
+    """A loader that is passed a function which does the loading.  The
+    function becomes the name of the template passed and has to return either
+    an unicode string with the template source, a tuple in the form ``(source,
+    filename, uptodatefunc)`` or `None` if the template does not exist.
+
+    >>> def load_template(name):
+    ...     if name == 'index.html':
+    ...         return '...'
+    ...
+    >>> loader = FunctionLoader(load_template)
+
+    The `uptodatefunc` is a function that is called if autoreload is enabled
+    and has to return `True` if the template is still up to date.  For more
+    details have a look at :meth:`BaseLoader.get_source` which has the same
+    return value.
+    """
+
+    def __init__(self, load_func):
+        self.load_func = load_func
+
+    def get_source(self, environment, template):
+        rv = self.load_func(template)
+        if rv is None:
+            raise TemplateNotFound(template)
+        elif isinstance(rv, basestring):
+            return rv, None, None
+        return rv
+
+
+class PrefixLoader(BaseLoader):
+    """A loader that is passed a dict of loaders where each loader is bound
+    to a prefix.  The prefix is delimited from the template by a slash per
+    default, which can be changed by setting the `delimiter` argument to
+    something else::
+
+        loader = PrefixLoader({
+            'app1':     PackageLoader('mypackage.app1'),
+            'app2':     PackageLoader('mypackage.app2')
+        })
+
+    By loading ``'app1/index.html'`` the file from the app1 package is loaded,
+    by loading ``'app2/index.html'`` the file from the second.
+    """
+
+    def __init__(self, mapping, delimiter='/'):
+        self.mapping = mapping
+        self.delimiter = delimiter
+
+    def get_source(self, environment, template):
+        try:
+            prefix, name = template.split(self.delimiter, 1)
+            loader = self.mapping[prefix]
+        except (ValueError, KeyError):
+            raise TemplateNotFound(template)
+        try:
+            return loader.get_source(environment, name)
+        except TemplateNotFound:
+            # re-raise the exception with the correct fileame here.
+            # (the one that includes the prefix)
+            raise TemplateNotFound(template)
+
+    def list_templates(self):
+        result = []
+        for prefix, loader in self.mapping.iteritems():
+            for template in loader.list_templates():
+                result.append(prefix + self.delimiter + template)
+        return result
+
+
+class ChoiceLoader(BaseLoader):
+    """This loader works like the `PrefixLoader` just that no prefix is
+    specified.  If a template could not be found by one loader the next one
+    is tried.
+
+    >>> loader = ChoiceLoader([
+    ...     FileSystemLoader('/path/to/user/templates'),
+    ...     FileSystemLoader('/path/to/system/templates')
+    ... ])
+
+    This is useful if you want to allow users to override builtin templates
+    from a different location.
+    """
+
+    def __init__(self, loaders):
+        self.loaders = loaders
+
+    def get_source(self, environment, template):
+        for loader in self.loaders:
+            try:
+                return loader.get_source(environment, template)
+            except TemplateNotFound:
+                pass
+        raise TemplateNotFound(template)
+
+    def list_templates(self):
+        found = set()
+        for loader in self.loaders:
+            found.update(loader.list_templates())
+        return sorted(found)
+
+
+class _TemplateModule(ModuleType):
+    """Like a normal module but with support for weak references"""
+
+
+class ModuleLoader(BaseLoader):
+    """This loader loads templates from precompiled templates.
+
+    Example usage:
+
+    >>> loader = ChoiceLoader([
+    ...     ModuleLoader('/path/to/compiled/templates'),
+    ...     FileSystemLoader('/path/to/templates')
+    ... ])
+    """
+
+    has_source_access = False
+
+    def __init__(self, path):
+        package_name = '_ambari_jinja2_module_templates_%x' % id(self)
+
+        # create a fake module that looks for the templates in the
+        # path given.
+        mod = _TemplateModule(package_name)
+        if isinstance(path, basestring):
+            path = [path]
+        else:
+            path = list(path)
+        mod.__path__ = path
+
+        sys.modules[package_name] = weakref.proxy(mod,
+            lambda x: sys.modules.pop(package_name, None))
+
+        # the only strong reference, the sys.modules entry is weak
+        # so that the garbage collector can remove it once the
+        # loader that created it goes out of business.
+        self.module = mod
+        self.package_name = package_name
+
+    @staticmethod
+    def get_template_key(name):
+        return 'tmpl_' + sha1(name.encode('utf-8')).hexdigest()
+
+    @staticmethod
+    def get_module_filename(name):
+        return ModuleLoader.get_template_key(name) + '.py'
+
+    @internalcode
+    def load(self, environment, name, globals=None):
+        key = self.get_template_key(name)
+        module = '%s.%s' % (self.package_name, key)
+        mod = getattr(self.module, module, None)
+        if mod is None:
+            try:
+                mod = __import__(module, None, None, ['root'])
+            except ImportError:
+                raise TemplateNotFound(name)
+
+            # remove the entry from sys.modules, we only want the attribute
+            # on the module object we have stored on the loader.
+            sys.modules.pop(module, None)
+
+        return environment.template_class.from_module_dict(
+            environment, mod.__dict__, globals)

http://git-wip-us.apache.org/repos/asf/ambari/blob/658360a5/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/meta.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/meta.py b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/meta.py
new file mode 100644
index 0000000..afe7771
--- /dev/null
+++ b/ambari-common/src/main/python/ambari_jinja2/ambari_jinja2/meta.py
@@ -0,0 +1,102 @@
+# -*- coding: utf-8 -*-
+"""
+    ambari_jinja2.meta
+    ~~~~~~~~~~~
+
+    This module implements various functions that exposes information about
+    templates that might be interesting for various kinds of applications.
+
+    :copyright: (c) 2010 by the Jinja Team, see AUTHORS for more details.
+    :license: BSD, see LICENSE for more details.
+"""
+from ambari_jinja2 import nodes
+from ambari_jinja2.compiler import CodeGenerator
+
+
+class TrackingCodeGenerator(CodeGenerator):
+    """We abuse the code generator for introspection."""
+
+    def __init__(self, environment):
+        CodeGenerator.__init__(self, environment, '<introspection>',
+                               '<introspection>')
+        self.undeclared_identifiers = set()
+
+    def write(self, x):
+        """Don't write."""
+
+    def pull_locals(self, frame):
+        """Remember all undeclared identifiers."""
+        self.undeclared_identifiers.update(frame.identifiers.undeclared)
+
+
+def find_undeclared_variables(ast):
+    """Returns a set of all variables in the AST that will be looked up from
+    the context at runtime.  Because at compile time it's not known which
+    variables will be used depending on the path the execution takes at
+    runtime, all variables are returned.
+
+    >>> from ambari_jinja2 import Environment, meta
+    >>> env = Environment()
+    >>> ast = env.parse('{% set foo = 42 %}{{ bar + foo }}')
+    >>> meta.find_undeclared_variables(ast)
+    set(['bar'])
+
+    .. admonition:: Implementation
+
+       Internally the code generator is used for finding undeclared variables.
+       This is good to know because the code generator might raise a
+       :exc:`TemplateAssertionError` during compilation and as a matter of
+       fact this function can currently raise that exception as well.
+    """
+    codegen = TrackingCodeGenerator(ast.environment)
+    codegen.visit(ast)
+    return codegen.undeclared_identifiers
+
+
+def find_referenced_templates(ast):
+    """Finds all the referenced templates from the AST.  This will return an
+    iterator over all the hardcoded template extensions, inclusions and
+    imports.  If dynamic inheritance or inclusion is used, `None` will be
+    yielded.
+
+    >>> from ambari_jinja2 import Environment, meta
+    >>> env = Environment()
+    >>> ast = env.parse('{% extends "layout.html" %}{% include helper %}')
+    >>> list(meta.find_referenced_templates(ast))
+    ['layout.html', None]
+
+    This function is useful for dependency tracking.  For example if you want
+    to rebuild parts of the website after a layout template has changed.
+    """
+    for node in ast.find_all((nodes.Extends, nodes.FromImport, nodes.Import,
+                              nodes.Include)):
+        if not isinstance(node.template, nodes.Const):
+            # a tuple with some non consts in there
+            if isinstance(node.template, (nodes.Tuple, nodes.List)):
+                for template_name in node.template.items:
+                    # something const, only yield the strings and ignore
+                    # non-string consts that really just make no sense
+                    if isinstance(template_name, nodes.Const):
+                        if isinstance(template_name.value, basestring):
+                            yield template_name.value
+                    # something dynamic in there
+                    else:
+                        yield None
+            # something dynamic we don't know about here
+            else:
+                yield None
+            continue
+        # constant is a basestring, direct template name
+        if isinstance(node.template.value, basestring):
+            yield node.template.value
+        # a tuple or list (latter *should* not happen) made of consts,
+        # yield the consts that are strings.  We could warn here for
+        # non string values
+        elif isinstance(node, nodes.Include) and \
+             isinstance(node.template.value, (tuple, list)):
+            for template_name in node.template.value:
+                if isinstance(template_name, basestring):
+                    yield template_name
+        # something else we don't care about, we could warn here
+        else:
+            yield None