ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject [1/5] AMBARI-6682. Add generate_config method to Script (aonishuk)
Date Fri, 01 Aug 2014 13:22:06 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk dd888838d -> 55bbcae46


http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-common/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/libraries/script/script.py
b/ambari-common/src/main/python/resource_management/libraries/script/script.py
new file mode 100644
index 0000000..bee2f36
--- /dev/null
+++ b/ambari-common/src/main/python/resource_management/libraries/script/script.py
@@ -0,0 +1,267 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+import tarfile
+import tempfile
+
+__all__ = ["Script"]
+
+import os
+import sys
+import json
+import logging
+from contextlib import closing
+
+
+from resource_management.libraries.resources import XmlConfig
+from resource_management.core.resources import File, Directory
+from resource_management.core.source import InlineTemplate
+
+from resource_management.core.environment import Environment
+from resource_management.core.exceptions import Fail, ClientComponentHasNoStatus, ComponentIsNotRunning
+from resource_management.core.resources.packaging import Package
+from resource_management.libraries.script.config_dictionary import ConfigDictionary
+
+
+USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT>
<LOGGING_LEVEL> <TMP_DIR>
+
+<COMMAND> command type (INSTALL/CONFIGURE/START/STOP/SERVICE_CHECK...)
+<JSON_CONFIG> path to command json file. Ex: /var/lib/ambari-agent/data/command-2.json
+<BASEDIR> path to service metadata dir. Ex: /var/lib/ambari-agent/cache/stacks/HDP/2.0.6/services/HDFS
+<STROUTPUT> path to file with structured command output (file will be created). Ex:/tmp/my.txt
+<LOGGING_LEVEL> log level for stdout. Ex:DEBUG,INFO
+<TMP_DIR> temporary directory for executable scripts. Ex: /var/lib/ambari-agent/data/tmp
+"""
+
+class Script(object):
+  """
+  Executes a command for custom service. stdout and stderr are written to
+  tmpoutfile and to tmperrfile respectively.
+  Script instances share configuration as a class parameter and therefore
+  different Script instances can not be used from different threads at
+  the same time within a single python process
+
+  Accepted command line arguments mapping:
+  1 command type (START/STOP/...)
+  2 path to command json file
+  3 path to service metadata dir (Directory "package" inside service directory)
+  4 path to file with structured command output (file will be created)
+  """
+  structuredOut = {}
+
+  def put_structured_out(self, sout):
+    Script.structuredOut.update(sout)
+    try:
+      with open(self.stroutfile, 'w') as fp:
+        json.dump(Script.structuredOut, fp)
+    except IOError:
+      Script.structuredOut.update({"errMsg" : "Unable to write to " + self.stroutfile})
+
+  def execute(self):
+    """
+    Sets up logging;
+    Parses command parameters and executes method relevant to command type
+    """
+    # set up logging (two separate loggers for stderr and stdout with different loglevels)
+    logger = logging.getLogger('resource_management')
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(asctime)s - %(message)s')
+    chout = logging.StreamHandler(sys.stdout)
+    chout.setLevel(logging.INFO)
+    chout.setFormatter(formatter)
+    cherr = logging.StreamHandler(sys.stderr)
+    cherr.setLevel(logging.ERROR)
+    cherr.setFormatter(formatter)
+    logger.addHandler(cherr)
+    logger.addHandler(chout)
+    
+    # parse arguments
+    if len(sys.argv) < 7: 
+     logger.error("Script expects at least 6 arguments")
+     print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
+     sys.exit(1)
+    
+    command_name = str.lower(sys.argv[1])
+    command_data_file = sys.argv[2]
+    basedir = sys.argv[3]
+    self.stroutfile = sys.argv[4]
+    logging_level = sys.argv[5]
+    Script.tmp_dir = sys.argv[6]
+
+    logging_level_str = logging._levelNames[logging_level]
+    chout.setLevel(logging_level_str)
+    logger.setLevel(logging_level_str)
+      
+    try:
+      with open(command_data_file, "r") as f:
+        pass
+        Script.config = ConfigDictionary(json.load(f))
+    except IOError:
+      logger.exception("Can not read json file with command parameters: ")
+      sys.exit(1)
+    # Run class method depending on a command type
+    try:
+      method = self.choose_method_to_execute(command_name)
+      with Environment(basedir) as env:
+        method(env)
+    except ClientComponentHasNoStatus or ComponentIsNotRunning:
+      # Support of component status checks.
+      # Non-zero exit code is interpreted as an INSTALLED status of a component
+      sys.exit(1)
+    except Fail:
+      logger.exception("Error while executing command '{0}':".format(command_name))
+      sys.exit(1)
+
+
+  def choose_method_to_execute(self, command_name):
+    """
+    Returns a callable object that should be executed for a given command.
+    """
+    self_methods = dir(self)
+    if not command_name in self_methods:
+      raise Fail("Script '{0}' has no method '{1}'".format(sys.argv[0], command_name))
+    method = getattr(self, command_name)
+    return method
+
+
+  @staticmethod
+  def get_config():
+    """
+    HACK. Uses static field to store configuration. This is a workaround for
+    "circular dependency" issue when importing params.py file and passing to
+     it a configuration instance.
+    """
+    return Script.config
+
+
+  @staticmethod
+  def get_tmp_dir():
+    """
+    HACK. Uses static field to avoid "circular dependency" issue when
+    importing params.py.
+    """
+    return Script.tmp_dir
+
+
+  def install(self, env):
+    """
+    Default implementation of install command is to install all packages
+    from a list, received from the server.
+    Feel free to override install() method with your implementation. It
+    usually makes sense to call install_packages() manually in this case
+    """
+    self.install_packages(env)
+
+
+  def install_packages(self, env, exclude_packages=[]):
+    """
+    List of packages that are required< by service is received from the server
+    as a command parameter. The method installs all packages
+    from this list
+    """
+    config = self.get_config()
+    
+    try:
+      package_list_str = config['hostLevelParams']['package_list']
+      if isinstance(package_list_str,basestring) and len(package_list_str) > 0:
+        package_list = json.loads(package_list_str)
+        for package in package_list:
+          if not package['name'] in exclude_packages:
+            name = package['name']
+            Package(name)
+    except KeyError:
+      pass # No reason to worry
+    
+    #RepoInstaller.remove_repos(config)
+
+
+
+  def fail_with_error(self, message):
+    """
+    Prints error message and exits with non-zero exit code
+    """
+    print("Error: " + message)
+    sys.stderr.write("Error: " + message)
+    sys.exit(1)
+
+  def start(self, env):
+    """
+    To be overridden by subclasses
+    """
+    self.fail_with_error('start method isn\'t implemented')
+
+  def stop(self, env):
+    """
+    To be overridden by subclasses
+    """
+    self.fail_with_error('stop method isn\'t implemented')
+
+  def restart(self, env):
+    """
+    Default implementation of restart command is to call stop and start methods
+    Feel free to override restart() method with your implementation.
+    For client components we call install
+    """
+    config = self.get_config()
+    componentCategory = None
+    try :
+      componentCategory = config['roleParams']['component_category']
+    except KeyError:
+      pass
+
+    if componentCategory and componentCategory.strip().lower() == 'CLIENT'.lower():
+      self.install(env)
+    else:
+      self.stop(env)
+      self.start(env)
+
+  def configure(self, env):
+    """
+    To be overridden by subclasses
+    """
+    self.fail_with_error('configure method isn\'t implemented')
+
+  def generate_configs(self, env):
+    """
+    Generates config files and stores them as an archive in tmp_dir
+    based on xml_configs_list and env_configs_list from commandParams
+    """
+    import params
+    config = self.get_config()
+    xml_configs_list = json.loads(config['commandParams']['xml_configs_list'])
+    env_configs_list = json.loads(config['commandParams']['env_configs_list'])
+    conf_tmp_dir = tempfile.mkdtemp()
+    output_filename = os.path.join(self.get_tmp_dir(),"client-configs.tar.gz")
+
+    Directory(self.get_tmp_dir(), recursive=True)
+    for file_dict in xml_configs_list:
+      for filename, dict in file_dict.iteritems():
+        XmlConfig(filename,
+                  conf_dir=conf_tmp_dir,
+                  configurations=params.config['configurations'][dict],
+                  configuration_attributes=params.config['configuration_attributes'][dict],
+        )
+    for file_dict in env_configs_list:
+      for filename,dict in file_dict.iteritems():
+        File(os.path.join(conf_tmp_dir, filename),
+             content=InlineTemplate(params.config['configurations'][dict]['content'])
+        )
+    with closing(tarfile.open(output_filename, "w:gz")) as tar:
+      tar.add(conf_tmp_dir, arcname=os.path.basename("."))
+    Directory(conf_tmp_dir, action="delete")

http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-server/conf/unix/install-helper.sh
----------------------------------------------------------------------
diff --git a/ambari-server/conf/unix/install-helper.sh b/ambari-server/conf/unix/install-helper.sh
index 740c2cc..dafe987 100644
--- a/ambari-server/conf/unix/install-helper.sh
+++ b/ambari-server/conf/unix/install-helper.sh
@@ -18,9 +18,11 @@
 ##################################################################
 
 COMMON_DIR="/usr/lib/python2.6/site-packages/ambari_commons"
+RESOURCE_MANAGEMENT_DIR="/usr/lib/python2.6/site-packages/resource_management"
 OLD_COMMON_DIR="/usr/lib/python2.6/site-packages/common_functions"
 INSTALL_HELPER_AGENT="/var/lib/ambari-agent/install-helper.sh"
 COMMON_DIR_SERVER="/usr/lib/ambari-server/lib/ambari_commons"
+RESOURCE_MANAGEMENT_DIR_SERVER="/usr/lib/ambari-server/lib/resource_management"
 
 PYTHON_WRAPER_TARGET="/usr/bin/ambari-python-wrap"
 PYTHON_WRAPER_SOURCE="/var/lib/ambari-server/ambari-python-wrap"
@@ -31,6 +33,10 @@ do_install(){
   if [ ! -d "$COMMON_DIR" ]; then
     ln -s "$COMMON_DIR_SERVER" "$COMMON_DIR"
   fi
+  # setting resource_management shared resource
+  if [ ! -d "$RESOURCE_MANAGEMENT_DIR" ]; then
+    ln -s "$RESOURCE_MANAGEMENT_DIR_SERVER" "$RESOURCE_MANAGEMENT_DIR"
+  fi
   # setting python-wrapper script
   if [ ! -f "$PYTHON_WRAPER_TARGET" ]; then
     ln -s "$PYTHON_WRAPER_SOURCE" "$PYTHON_WRAPER_TARGET"
@@ -40,6 +46,7 @@ do_install(){
 do_remove(){
 
   rm -rf "$COMMON_DIR"
+  rm -rf "$RESOURCE_MANAGEMENT_DIR"
 
   if [ -f "$PYTHON_WRAPER_TARGET" ]; then
     rm -f "$PYTHON_WRAPER_TARGET"

http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-server/pom.xml
----------------------------------------------------------------------
diff --git a/ambari-server/pom.xml b/ambari-server/pom.xml
index a6d03ab..1c5f5d5 100644
--- a/ambari-server/pom.xml
+++ b/ambari-server/pom.xml
@@ -32,6 +32,7 @@
     <hdpUrlForCentos6>http://public-repo-1.hortonworks.com/HDP/centos6/2.x/updates/2.1.1.0</hdpUrlForCentos6>
     <hdpLatestUrl>http://public-repo-1.hortonworks.com/HDP/hdp_urlinfo.json</hdpLatestUrl>
     <ambari_commons.install.dir>/usr/lib/ambari-server/lib/ambari_commons</ambari_commons.install.dir>
+    <resource_management.install.dir>/usr/lib/ambari-server/lib/resource_management</resource_management.install.dir>
     <ambari-web-dir>${basedir}/../ambari-web/public</ambari-web-dir>
     <ambari-admin-dir>${basedir}/../ambari-admin</ambari-admin-dir>
   </properties>
@@ -264,6 +265,16 @@
                 </source>
               </sources>
             </mapping>
+            <mapping>
+              <directory>${resource_management.install.dir}</directory>
+              <sources>
+                <source>
+                  <location>
+                    ${project.basedir}/../ambari-common/src/main/python/resource_management
+                  </location>
+                </source>
+              </sources>
+            </mapping>
               <mapping>
               <directory>/usr/sbin</directory>
               <filemode>755</filemode>
@@ -864,6 +875,19 @@
                 <group>root</group>
               </mapper>
             </data>
+            <data>
+              <src>
+                ${project.basedir}/../ambari-common/src/main/python/resource_management
+              </src>
+              <type>directory</type>
+              <mapper>
+                <type>perm</type>
+                <prefix>${resource_management.install.dir}</prefix>
+                <filemode>755</filemode>
+                <user>root</user>
+                <group>root</group>
+              </mapper>
+            </data>
           </dataSet>
         </configuration>
       </plugin>
@@ -897,7 +921,7 @@
                 <argument>${custom.tests}</argument>
               </arguments>
               <environmentVariables>
-                  <PYTHONPATH>${project.basedir}/../ambari-agent/src/main/python:${project.basedir}/../ambari-common/src/main/python/jinja2:${project.basedir}/../ambari-common/src/main/python:${project.basedir}/../ambari-common/src/main/python/ambari_commons:${project.basedir}/../ambari-common/src/test/python:${project.basedir}/src/main/python:${project.basedir}/src/main/python/ambari-server-state:${project.basedir}/src/test/python:$PYTHONPATH</PYTHONPATH>
+                  <PYTHONPATH>${project.basedir}/../ambari-agent/src/main/python:${project.basedir}/../ambari-common/src/main/python/jinja2:${project.basedir}/../ambari-common/src/main/python:${project.basedir}/../ambari-common/src/main/python/ambari_commons:${project.basedir}/../ambari-common/src/main/python/resource_management:${project.basedir}/../ambari-common/src/test/python:${project.basedir}/src/main/python:${project.basedir}/src/main/python/ambari-server-state:${project.basedir}/src/test/python:$PYTHONPATH</PYTHONPATH>
               </environmentVariables>
               <skip>${skipTests}</skip>
             </configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
new file mode 100644
index 0000000..2f2ca5a
--- /dev/null
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_hdfs_client.py
@@ -0,0 +1,54 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+from mock.mock import MagicMock, patch
+import tempfile
+import tarfile
+import contextlib
+from resource_management import *
+from stacks.utils.RMFTestCase import *
+
+
+@patch.object(tarfile,"open", new = MagicMock())
+@patch.object(tempfile,"mkdtemp", new = MagicMock(return_value='/tmp/123'))
+@patch.object(contextlib,"closing", new = MagicMock())
+@patch("os.path.exists", new = MagicMock(return_value=True))
+class TestDatanode(RMFTestCase):
+
+  def test_generate_configs_default(self):
+    self.executeScript("2.0.6/services/HDFS/package/scripts/hdfs_client.py",
+                       classname = "HdfsClient",
+                       command = "generate_configs",
+                       config_file="default.json"
+    )
+    self.assertResourceCalled('Directory', '/tmp',
+                              recursive = True,
+                              )
+    self.assertResourceCalled('XmlConfig', 'hdfs-site.xml',
+                              conf_dir = '/tmp/123',
+                              configuration_attributes = self.getConfig()['configuration_attributes']['hdfs-site'],
+                              configurations = self.getConfig()['configurations']['hdfs-site'],
+                              )
+    self.assertResourceCalled('File', '/tmp/123/hadoop-env.sh',
+                              content = InlineTemplate(self.getConfig()['configurations']['hadoop-env']['content']),
+                              )
+    self.assertResourceCalled('Directory', '/tmp/123',
+                              action = ['delete'],
+                              )
+    self.assertNoMoreResources()

http://git-wip-us.apache.org/repos/asf/ambari/blob/55bbcae4/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
index cc69ef8..beeb3e7 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
+++ b/ambari-server/src/test/python/stacks/2.0.6/configs/default.json
@@ -26,7 +26,10 @@
         "script_type": "PYTHON",
         "script": "scripts/service_check.py",
         "excluded_hosts": "host1,host2",
-        "mark_draining_only" : "false"
+        "mark_draining_only" : "false",
+        "xml_configs_list":"[{\"hdfs-site.xml\":\"hdfs-site\"}]",
+        "env_configs_list":"[{\"hadoop-env.sh\":\"hadoop-env\"}]"
+
     },
     "taskId": 152, 
     "public_hostname": "c6401.ambari.apache.org", 


Mime
View raw message