Return-Path: X-Original-To: apmail-incubator-ambari-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-ambari-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id B3608CBD9 for ; Fri, 1 Nov 2013 15:29:04 +0000 (UTC) Received: (qmail 65880 invoked by uid 500); 1 Nov 2013 15:28:27 -0000 Delivered-To: apmail-incubator-ambari-commits-archive@incubator.apache.org Received: (qmail 65848 invoked by uid 500); 1 Nov 2013 15:28:26 -0000 Mailing-List: contact ambari-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@incubator.apache.org Delivered-To: mailing list ambari-commits@incubator.apache.org Received: (qmail 65402 invoked by uid 99); 1 Nov 2013 15:28:13 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 01 Nov 2013 15:28:13 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id E681AD833; Fri, 1 Nov 2013 15:28:12 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: dmitriusan@apache.org To: ambari-commits@incubator.apache.org Date: Fri, 01 Nov 2013 15:28:24 -0000 Message-Id: <7bc5ff087dc64b23a57136ce25d1e8e8@git.apache.org> In-Reply-To: <8549d1033a064cfc89c01cb3a9663d30@git.apache.org> References: <8549d1033a064cfc89c01cb3a9663d30@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [13/15] git commit: AMBARI-3663. Resource Management. Easy formatting of configurations (Andrew Onischuk via dlysnichenko) AMBARI-3663. Resource Management. Easy formatting of configurations (Andrew Onischuk via dlysnichenko) Project: http://git-wip-us.apache.org/repos/asf/incubator-ambari/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-ambari/commit/002060f8 Tree: http://git-wip-us.apache.org/repos/asf/incubator-ambari/tree/002060f8 Diff: http://git-wip-us.apache.org/repos/asf/incubator-ambari/diff/002060f8 Branch: refs/heads/trunk Commit: 002060f84dc18cdb03c38924b3fe5287f07c8d96 Parents: fa817f6 Author: Lisnichenko Dmitro Authored: Fri Nov 1 17:23:57 2013 +0200 Committer: Lisnichenko Dmitro Committed: Fri Nov 1 17:23:57 2013 +0200 ---------------------------------------------------------------------- .../python/resource_management/core/base.py | 7 ++-- .../resource_management/core/environment.py | 38 +++++++++++--------- .../python/resource_management/core/source.py | 32 +++++++++-------- .../python/resource_management/core/utils.py | 36 +++++++------------ .../libraries/providers/execute_hadoop.py | 24 ++++++------- 5 files changed, 65 insertions(+), 72 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/002060f8/ambari-agent/src/main/python/resource_management/core/base.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/resource_management/core/base.py b/ambari-agent/src/main/python/resource_management/core/base.py index fb2e26d..0ad34ce 100644 --- a/ambari-agent/src/main/python/resource_management/core/base.py +++ b/ambari-agent/src/main/python/resource_management/core/base.py @@ -5,7 +5,7 @@ __all__ = ["Resource", "ResourceArgument", "ForcedListArgument", import logging from resource_management.core.exceptions import Fail, InvalidArgument -from resource_management.core.environment import Environment, Substitutor +from resource_management.core.environment import Environment class ResourceArgument(object): def __init__(self, default=None, required=False, allow_override=False): @@ -95,7 +95,6 @@ class Resource(object): name = name[0] env = env or Environment.get_instance() - name = Substitutor.substitute(name) provider = provider or getattr(cls, 'provider', None) r_type = cls.__name__ @@ -123,7 +122,7 @@ class Resource(object): return self.env = env or Environment.get_instance() - self.name = Substitutor.substitute(name) + self.name = name self.provider = provider or getattr(self, 'provider', None) @@ -135,7 +134,7 @@ class Resource(object): raise Fail("%s received unsupported argument %s" % (self, key)) else: try: - self.arguments[key] = Substitutor.substitute(arg.validate(value)) + self.arguments[key] = arg.validate(value) except InvalidArgument, exc: raise InvalidArgument("%s %s" % (self, exc)) http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/002060f8/ambari-agent/src/main/python/resource_management/core/environment.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/resource_management/core/environment.py b/ambari-agent/src/main/python/resource_management/core/environment.py index d6557b0..fb0536d 100644 --- a/ambari-agent/src/main/python/resource_management/core/environment.py +++ b/ambari-agent/src/main/python/resource_management/core/environment.py @@ -1,19 +1,20 @@ #!/usr/bin/env python -__all__ = ["Environment"] +__all__ = ["Environment","format"] import logging +import sys import os import shutil import time from datetime import datetime +from string import Formatter from resource_management.core import shell from resource_management.core.exceptions import Fail from resource_management.core.providers import find_provider -from resource_management.core.utils import AttributeDictionary, ParamsAttributeDictionary +from resource_management.core.utils import AttributeDictionary, checked_unite from resource_management.core.system import System -from string import Template class Environment(object): @@ -26,6 +27,7 @@ class Environment(object): @param params: configurations dictionary (this will be accessible in the templates) """ self.log = logging.getLogger("resource_management") + self.formatter = ConfigurationFormatter() self.reset(basedir, params) def reset(self, basedir, params): @@ -44,7 +46,7 @@ class Environment(object): # dir where templates,failes dirs are 'basedir': basedir, # variables, which can be used in templates - 'params': ParamsAttributeDictionary(Substitutor, params.copy()), + 'params': params.copy(), }) def backup_file(self, path): @@ -169,18 +171,20 @@ class Environment(object): self.resource_list = state['resource_list'] self.delayed_actions = state['delayed_actions'] - -class Substitutor(): - log = logging.getLogger("resource_management.resource") - @staticmethod - def substitute(val): +class ConfigurationFormatter(Formatter): + def format(self, format_string, *args, **kwargs): env = Environment.get_instance() - dic = env.config.params + variables = kwargs + params = env.config.params - if dic and isinstance(val, str): - result = Template(val).substitute(dic) - if '$' in val: - Substitutor.log.debug("%s after substitution is %s", val, result) - return result - - return val + result = checked_unite(variables, params) + return self.vformat(format_string, args, result) + +def format(format_string, *args, **kwargs): + env = Environment.get_instance() + variables = sys._getframe(1).f_locals + + result = checked_unite(kwargs, variables) + result.pop("self", None) # self kwarg would result in an error + return env.formatter.format(format_string, args, **result) + http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/002060f8/ambari-agent/src/main/python/resource_management/core/source.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/resource_management/core/source.py b/ambari-agent/src/main/python/resource_management/core/source.py index 2fcd395..b27bacb 100644 --- a/ambari-agent/src/main/python/resource_management/core/source.py +++ b/ambari-agent/src/main/python/resource_management/core/source.py @@ -1,5 +1,6 @@ from __future__ import with_statement -from resource_management.core.environment import Substitutor, Environment +from resource_management.core.environment import Environment +from resource_management.core.utils import checked_unite __all__ = ["Source", "Template", "InlineTemplate", "StaticFile", "DownloadSource"] @@ -10,9 +11,9 @@ import urlparse class Source(object): - def __init__(self, name, env=None): - self.name = Substitutor.substitute(name) - self.env = env or Environment.get_instance() + def __init__(self, name): + self.env = Environment.get_instance() + self.name = name def get_content(self): raise NotImplementedError() @@ -25,8 +26,8 @@ class Source(object): class StaticFile(Source): - def __init__(self, name, env=None): - super(StaticFile, self).__init__(name, env) + def __init__(self, name): + super(StaticFile, self).__init__(name) def get_content(self): # absolute path @@ -42,7 +43,7 @@ class StaticFile(Source): try: - from jinja2 import Environment as JinjaEnvironment, BaseLoader, TemplateNotFound, FunctionLoader + from jinja2 import Environment as JinjaEnvironment, BaseLoader, TemplateNotFound, FunctionLoader, StrictUndefined except ImportError: class Template(Source): def __init__(self, name, variables=None, env=None): @@ -73,14 +74,17 @@ else: return source, path, lambda: mtime == os.path.getmtime(path) class Template(Source): - def __init__(self, name, variables=None, env=None): - super(Template, self).__init__(name, env) + def __init__(self, name, **kwargs): + """ + @param kwargs: Additional variables passed to template + """ + super(Template, self).__init__(name) params = self.env.config.params - variables = params if params else variables + variables = checked_unite(params, kwargs) self.context = variables.copy() if variables else {} if not hasattr(self, 'template_env'): self.template_env = JinjaEnvironment(loader=TemplateLoader(self.env), - autoescape=False) + autoescape=False, undefined=StrictUndefined) self.template = self.template_env.get_template(self.name) def get_content(self): @@ -94,13 +98,13 @@ else: return rendered + "\n" if not rendered.endswith('\n') else rendered class InlineTemplate(Template): - def __init__(self, name, variables=None, env=None): + def __init__(self, name, **kwargs): self.template_env = JinjaEnvironment(loader=FunctionLoader(lambda text: text)) - super(InlineTemplate, self).__init__(name, variables, env) + super(InlineTemplate, self).__init__(name, **kwargs) class DownloadSource(Source): - def __init__(self, name, cache=True, md5sum=None, env=None): + def __init__(self, name, cache=True, md5sum=None): super(DownloadSource, self).__init__(name) self.url = self.name self.md5sum = md5sum http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/002060f8/ambari-agent/src/main/python/resource_management/core/utils.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/resource_management/core/utils.py b/ambari-agent/src/main/python/resource_management/core/utils.py index 4cd5207..028b911 100644 --- a/ambari-agent/src/main/python/resource_management/core/utils.py +++ b/ambari-agent/src/main/python/resource_management/core/utils.py @@ -54,7 +54,10 @@ class AttributeDictionary(object): return self._dict.__repr__() def __unicode__(self): - return self._dict.__unicode__() + if isinstance(self._dict, str): + return self._dict.__unicode__() + else: + return str(self._dict) def __str__(self): return self._dict.__str__() @@ -68,27 +71,12 @@ class AttributeDictionary(object): def __setstate__(self, state): super(AttributeDictionary, self).__setattr__("_dict", state) -class ParamsAttributeDictionary(AttributeDictionary): - """ - This class can store user parameters - and it supports some features necessary for substitution to work. - """ - def __init__(self, substitutor, *args, **kwargs): - super(ParamsAttributeDictionary, self).__init__(*args, **kwargs) - super(AttributeDictionary, self).__setattr__("substitutor", substitutor) - - def __getitem__(self, name): - try: - return self._convert_value(self._dict[name]) - except KeyError as ex: - raise Fail("Configuration $%s not found!" % str(ex).strip("'")) +def checked_unite(dict1, dict2): + for key in dict1: + if key in dict2: + raise Fail("Variable '%s' already exists more than once as a variable/configuration/kwarg parameter. Cannot evaluate it." % key) - def copy(self): - # don't allow real copying to be able to change params passed to jinja2 - return self - - def __unicode__(self): - if isinstance(self._dict, str): - return self._dict.__unicode__() - else: - return str(self._dict) \ No newline at end of file + result = dict1.copy() + result.update(dict2) + + return result \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-ambari/blob/002060f8/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py ---------------------------------------------------------------------- diff --git a/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py b/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py index 1fbfab0..39788bb 100644 --- a/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py +++ b/ambari-agent/src/main/python/resource_management/libraries/providers/execute_hadoop.py @@ -2,21 +2,19 @@ from resource_management import * class ExecuteHadoopProvider(Provider): def action_run(self): - if self.resource.security_enabled and not self.resource.kinit_override: - kinit_if_needed = "su - {user} -c '{kinit_path_local} -kt {keytab} {principal}'".format( - user=self.resource.user, kinit_path_local=self.resource.kinit_path_local, - keytab=self.resource.keytab, principal=self.resource.principal) - else: - kinit_if_needed = "" - - cmd = "hadoop --config {conf_dir} {command}".format(conf_dir=self.resource.conf_dir, command=self.resource.command) + kinit_path_local = self.resource.kinit_path_local + keytab = self.resource.keytab + principal = self.resource.principal + conf_dir = self.resource.conf_dir + command = self.resource.command - if kinit_if_needed != "": - Execute ( kinit_if_needed, - path = ['/bin'] + if self.resource.security_enabled and not self.resource.kinit_override: + Execute ((kinit_path_local, '-kt', keytab, principal), + path = ['/bin'], + user = self.resource.user ) - - Execute ( cmd, + + Execute (('hadoop', '--config', conf_dir, command), user = self.resource.user, tries = self.resource.tries, try_sleep = self.resource.try_sleep,