ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From yus...@apache.org
Subject [42/51] [partial] AMBARI-7621. Import initial contribution for Ambari support on Windows to branch-windows-dev. (Jayush Luniya and Florian Barca via yusaku)
Date Tue, 07 Oct 2014 22:53:10 GMT
http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/main/python/resource_management/libraries/script/script.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/script/script.py b/ambari-agent/src/main/python/resource_management/libraries/script/script.py
new file mode 100644
index 0000000..9ce8b1b
--- /dev/null
+++ b/ambari-agent/src/main/python/resource_management/libraries/script/script.py
@@ -0,0 +1,251 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+__all__ = ["Script"]
+
+import os
+import sys
+import json
+import logging
+import platform
+
+from resource_management.core.environment import Environment
+from resource_management.core.exceptions import Fail, ClientComponentHasNoStatus, ComponentIsNotRunning
+from resource_management.core.resources.packaging import Package
+from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration
+from resource_management.libraries.functions.install_hdp_msi import install_windows_msi
+from resource_management.libraries.functions.reload_windows_env import reload_windows_env
+USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEVEL>
+
+<COMMAND> command type (INSTALL/CONFIGURE/START/STOP/SERVICE_CHECK...)
+<JSON_CONFIG> path to command json file. Ex: /var/lib/ambari-agent/data/command-2.json
+<BASEDIR> path to service metadata dir. Ex: /var/lib/ambari-agent/cache/stacks/HDP/2.0.6/services/HDFS
+<STROUTPUT> path to file with structured command output (file will be created). Ex:/tmp/my.txt
+<LOGGING_LEVEL> log level for stdout. Ex:DEBUG,INFO
+"""
+
+_PASSWORD_MAP = {"/configurations/hadoop-env/hadoop.user.name":"/configurations/hadoop-env/hadoop.user.password"}
+
+def get_path_form_configuration(name, configuration):
+  subdicts = filter(None, name.split('/'))
+
+  for x in subdicts:
+    if x in configuration:
+      configuration = configuration[x]
+    else:
+      return None
+
+  return configuration
+
+class Script(object):
+  """
+  Executes a command for custom service. stdout and stderr are written to
+  tmpoutfile and to tmperrfile respectively.
+  Script instances share configuration as a class parameter and therefore
+  different Script instances can not be used from different threads at
+  the same time within a single python process
+
+  Accepted command line arguments mapping:
+  1 command type (START/STOP/...)
+  2 path to command json file
+  3 path to service metadata dir (Directory "package" inside service directory)
+  4 path to file with structured command output (file will be created)
+  """
+  structuredOut = {}
+
+  def put_structured_out(self, sout):
+    Script.structuredOut.update(sout)
+    try:
+      with open(self.stroutfile, 'w') as fp:
+        json.dump(Script.structuredOut, fp)
+    except IOError:
+      Script.structuredOut.update({"errMsg" : "Unable to write to " + self.stroutfile})
+
+  def execute(self):
+    """
+    Sets up logging;
+    Parses command parameters and executes method relevant to command type
+    """
+    # set up logging (two separate loggers for stderr and stdout with different loglevels)
+    logger = logging.getLogger('resource_management')
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(asctime)s - %(message)s')
+    chout = logging.StreamHandler(sys.stdout)
+    chout.setLevel(logging.INFO)
+    chout.setFormatter(formatter)
+    cherr = logging.StreamHandler(sys.stderr)
+    cherr.setLevel(logging.ERROR)
+    cherr.setFormatter(formatter)
+    logger.addHandler(cherr)
+    logger.addHandler(chout)
+    
+    # parse arguments
+    if len(sys.argv) < 6: 
+     logger.error("Script expects at least 5 arguments")
+     print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
+     sys.exit(1)
+    
+    command_name = str.lower(sys.argv[1])
+    command_data_file = sys.argv[2]
+    basedir = sys.argv[3]
+    self.stroutfile = sys.argv[4]
+    logging_level = sys.argv[5]
+    
+    logging_level_str = logging._levelNames[logging_level]
+    chout.setLevel(logging_level_str)
+    logger.setLevel(logging_level_str)
+
+    # on windows we need to reload some of env variables manually because there is no default paths for configs(like
+    # /etc/something/conf on linux. When this env vars created by one of the Script execution, they can not be updated
+    # in agent, so other Script executions will not be able to access to new env variables
+    if platform.system() == "Windows":
+      reload_windows_env()
+
+    try:
+      with open(command_data_file, "r") as f:
+        pass
+        Script.config = ConfigDictionary(json.load(f))
+        #load passwords here(used on windows to impersonate different users)
+        Script.passwords = {}
+        for k, v in _PASSWORD_MAP.iteritems():
+          if get_path_form_configuration(k,Script.config) and get_path_form_configuration(v,Script.config ):
+            Script.passwords[get_path_form_configuration(k,Script.config)] = get_path_form_configuration(v,Script.config)
+
+    except IOError:
+      logger.exception("Can not read json file with command parameters: ")
+      sys.exit(1)
+    # Run class method depending on a command type
+    try:
+      method = self.choose_method_to_execute(command_name)
+      with Environment(basedir) as env:
+        method(env)
+    except ClientComponentHasNoStatus or ComponentIsNotRunning:
+      # Support of component status checks.
+      # Non-zero exit code is interpreted as an INSTALLED status of a component
+      sys.exit(1)
+    except Fail:
+      logger.exception("Error while executing command '{0}':".format(command_name))
+      sys.exit(1)
+
+
+  def choose_method_to_execute(self, command_name):
+    """
+    Returns a callable object that should be executed for a given command.
+    """
+    self_methods = dir(self)
+    if not command_name in self_methods:
+      raise Fail("Script '{0}' has no method '{1}'".format(sys.argv[0], command_name))
+    method = getattr(self, command_name)
+    return method
+
+
+  @staticmethod
+  def get_config():
+    """
+    HACK. Uses static field to store configuration. This is a workaround for
+    "circular dependency" issue when importing params.py file and passing to
+     it a configuration instance.
+    """
+    return Script.config
+
+  @staticmethod
+  def get_password(user):
+    return Script.passwords[user]
+
+  def install(self, env):
+    """
+    Default implementation of install command is to install all packages
+    from a list, received from the server.
+    Feel free to override install() method with your implementation. It
+    usually makes sense to call install_packages() manually in this case
+    """
+    self.install_packages(env)
+
+
+  def install_packages(self, env, exclude_packages=[]):
+    """
+    List of packages that are required< by service is received from the server
+    as a command parameter. The method installs all packages
+    from this list
+    """
+    config = self.get_config()
+    if platform.system() == "Windows":
+      install_windows_msi(os.path.join(config['hostLevelParams']['jdk_location'], "hdp.msi"),
+                          config["hostLevelParams"]["agentCacheDir"], "hdp.msi", self.get_password("hadoop"))
+    else:
+      try:
+        package_list_str = config['hostLevelParams']['package_list']
+        if isinstance(package_list_str, basestring) and len(package_list_str) > 0:
+          package_list = json.loads(package_list_str)
+          for package in package_list:
+            if not package['name'] in exclude_packages:
+              name = package['name']
+              Package(name)
+      except KeyError:
+        pass  # No reason to worry
+
+        # RepoInstaller.remove_repos(config)
+
+
+
+  def fail_with_error(self, message):
+    """
+    Prints error message and exits with non-zero exit code
+    """
+    print("Error: " + message)
+    sys.stderr.write("Error: " + message)
+    sys.exit(1)
+
+  def start(self, env):
+    """
+    To be overridden by subclasses
+    """
+    self.fail_with_error('start method isn\'t implemented')
+
+  def stop(self, env):
+    """
+    To be overridden by subclasses
+    """
+    self.fail_with_error('stop method isn\'t implemented')
+
+  def restart(self, env):
+    """
+    Default implementation of restart command is to call stop and start methods
+    Feel free to override restart() method with your implementation.
+    For client components we call install
+    """
+    config = self.get_config()
+    componentCategory = None
+    try :
+      componentCategory = config['roleParams']['component_category']
+    except KeyError:
+      pass
+
+    if componentCategory and componentCategory.strip().lower() == 'CLIENT'.lower():
+      self.install(env)
+    else:
+      self.stop(env)
+      self.start(env)
+
+  def configure(self, env):
+    """
+    To be overridden by subclasses
+    """
+    self.fail_with_error('configure method isn\'t implemented')

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/main/python/resource_management/libraries/script/script.py.orig
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/resource_management/libraries/script/script.py.orig b/ambari-agent/src/main/python/resource_management/libraries/script/script.py.orig
new file mode 100644
index 0000000..df666f9
--- /dev/null
+++ b/ambari-agent/src/main/python/resource_management/libraries/script/script.py.orig
@@ -0,0 +1,250 @@
+#!/usr/bin/env python
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+__all__ = ["Script"]
+
+import os
+import sys
+import json
+import logging
+import platform
+
+from resource_management.core.environment import Environment
+from resource_management.core.exceptions import Fail, ClientComponentHasNoStatus, ComponentIsNotRunning
+from resource_management.core.resources.packaging import Package
+from resource_management.libraries.script.config_dictionary import ConfigDictionary, UnknownConfiguration
+from resource_management.libraries.functions.install_hdp_msi import install_windows_msi
+
+
+USAGE = """Usage: {0} <COMMAND> <JSON_CONFIG> <BASEDIR> <STROUTPUT> <LOGGING_LEVEL>
+
+<COMMAND> command type (INSTALL/CONFIGURE/START/STOP/SERVICE_CHECK...)
+<JSON_CONFIG> path to command json file. Ex: /var/lib/ambari-agent/data/command-2.json
+<BASEDIR> path to service metadata dir. Ex: /var/lib/ambari-agent/cache/stacks/HDP/2.0.6/services/HDFS
+<STROUTPUT> path to file with structured command output (file will be created). Ex:/tmp/my.txt
+<LOGGING_LEVEL> log level for stdout. Ex:DEBUG,INFO
+"""
+
+_PASSWORD_MAP = {"/configurations/hadoop-env/hadoop.user.name":"/configurations/hadoop-env/hadoop.user.password"}
+
+def get_path_form_configuration(name, configuration):
+  subdicts = filter(None, name.split('/'))
+
+  for x in subdicts:
+    if x in configuration:
+      configuration = configuration[x]
+    else:
+      return None
+
+  return configuration
+
+class Script(object):
+  """
+  Executes a command for custom service. stdout and stderr are written to
+  tmpoutfile and to tmperrfile respectively.
+  Script instances share configuration as a class parameter and therefore
+  different Script instances can not be used from different threads at
+  the same time within a single python process
+
+  Accepted command line arguments mapping:
+  1 command type (START/STOP/...)
+  2 path to command json file
+  3 path to service metadata dir (Directory "package" inside service directory)
+  4 path to file with structured command output (file will be created)
+  """
+  structuredOut = {}
+
+  def put_structured_out(self, sout):
+    Script.structuredOut.update(sout)
+    try:
+      with open(self.stroutfile, 'w') as fp:
+        json.dump(Script.structuredOut, fp)
+    except IOError:
+      Script.structuredOut.update({"errMsg" : "Unable to write to " + self.stroutfile})
+
+  def execute(self):
+    """
+    Sets up logging;
+    Parses command parameters and executes method relevant to command type
+    """
+    # set up logging (two separate loggers for stderr and stdout with different loglevels)
+    logger = logging.getLogger('resource_management')
+    logger.setLevel(logging.DEBUG)
+    formatter = logging.Formatter('%(asctime)s - %(message)s')
+    chout = logging.StreamHandler(sys.stdout)
+    chout.setLevel(logging.INFO)
+    chout.setFormatter(formatter)
+    cherr = logging.StreamHandler(sys.stderr)
+    cherr.setLevel(logging.ERROR)
+    cherr.setFormatter(formatter)
+    logger.addHandler(cherr)
+    logger.addHandler(chout)
+    
+    # parse arguments
+    if len(sys.argv) < 6: 
+     logger.error("Script expects at least 5 arguments")
+     print USAGE.format(os.path.basename(sys.argv[0])) # print to stdout
+     sys.exit(1)
+    
+    command_name = str.lower(sys.argv[1])
+    command_data_file = sys.argv[2]
+    basedir = sys.argv[3]
+    self.stroutfile = sys.argv[4]
+    logging_level = sys.argv[5]
+    
+    logging_level_str = logging._levelNames[logging_level]
+    chout.setLevel(logging_level_str)
+    logger.setLevel(logging_level_str)
+      
+    try:
+      with open(command_data_file, "r") as f:
+        pass
+        Script.config = ConfigDictionary(json.load(f))
+        #load passwords here(used on windows to impersonate different users)
+        Script.passwords = {}
+        for k, v in _PASSWORD_MAP.iteritems():
+          if get_path_form_configuration(k,Script.config) and get_path_form_configuration(v,Script.config ):
+            Script.passwords[get_path_form_configuration(k,Script.config)] = get_path_form_configuration(v,Script.config)
+
+    except IOError:
+      logger.exception("Can not read json file with command parameters: ")
+      sys.exit(1)
+    # Run class method depending on a command type
+    try:
+      method = self.choose_method_to_execute(command_name)
+      with Environment(basedir) as env:
+        method(env)
+    except ClientComponentHasNoStatus or ComponentIsNotRunning:
+      # Support of component status checks.
+      # Non-zero exit code is interpreted as an INSTALLED status of a component
+      sys.exit(1)
+    except Fail:
+      logger.exception("Error while executing command '{0}':".format(command_name))
+      sys.exit(1)
+
+
+  def choose_method_to_execute(self, command_name):
+    """
+    Returns a callable object that should be executed for a given command.
+    """
+    self_methods = dir(self)
+    if not command_name in self_methods:
+      raise Fail("Script '{0}' has no method '{1}'".format(sys.argv[0], command_name))
+    method = getattr(self, command_name)
+    return method
+
+
+  @staticmethod
+  def get_config():
+    """
+    HACK. Uses static field to store configuration. This is a workaround for
+    "circular dependency" issue when importing params.py file and passing to
+     it a configuration instance.
+    """
+    return Script.config
+
+  @staticmethod
+  def get_password(user):
+    return Script.passwords[user]
+
+  def install(self, env):
+    """
+    Default implementation of install command is to install all packages
+    from a list, received from the server.
+    Feel free to override install() method with your implementation. It
+    usually makes sense to call install_packages() manually in this case
+    """
+    self.install_packages(env)
+
+
+  def install_packages(self, env, exclude_packages=[]):
+    """
+    List of packages that are required< by service is received from the server
+    as a command parameter. The method installs all packages
+    from this list
+    """
+    config = self.get_config()
+    if platform.system() == "Windows":
+      install_windows_msi(os.path.join(config['hostLevelParams']['jdk_location'], "hdp.msi"),
+<<<<<<< HEAD
+                          config["hostLevelParams"]["agentCacheDir"], "hdp.msi")
+=======
+                          config["hostLevelParams"]["agentCacheDir"], "hdp.msi", self.get_password("hadoop"))
+>>>>>>> da12124c4425f3f87e15c06795c51cbde97beacb
+    else:
+      try:
+        package_list_str = config['hostLevelParams']['package_list']
+        if isinstance(package_list_str, basestring) and len(package_list_str) > 0:
+          package_list = json.loads(package_list_str)
+          for package in package_list:
+            if not package['name'] in exclude_packages:
+              name = package['name']
+              Package(name)
+      except KeyError:
+        pass  # No reason to worry
+
+        # RepoInstaller.remove_repos(config)
+
+
+
+  def fail_with_error(self, message):
+    """
+    Prints error message and exits with non-zero exit code
+    """
+    print("Error: " + message)
+    sys.stderr.write("Error: " + message)
+    sys.exit(1)
+
+  def start(self, env):
+    """
+    To be overridden by subclasses
+    """
+    self.fail_with_error('start method isn\'t implemented')
+
+  def stop(self, env):
+    """
+    To be overridden by subclasses
+    """
+    self.fail_with_error('stop method isn\'t implemented')
+
+  def restart(self, env):
+    """
+    Default implementation of restart command is to call stop and start methods
+    Feel free to override restart() method with your implementation.
+    For client components we call install
+    """
+    config = self.get_config()
+    componentCategory = None
+    try :
+      componentCategory = config['roleParams']['component_category']
+    except KeyError:
+      pass
+
+    if componentCategory and componentCategory.strip().lower() == 'CLIENT'.lower():
+      self.install(env)
+    else:
+      self.stop(env)
+      self.start(env)
+
+  def configure(self, env):
+    """
+    To be overridden by subclasses
+    """
+    self.fail_with_error('configure method isn\'t implemented')

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/main/python/setup.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/main/python/setup.py b/ambari-agent/src/main/python/setup.py
index 41c2097..69f6490 100644
--- a/ambari-agent/src/main/python/setup.py
+++ b/ambari-agent/src/main/python/setup.py
@@ -32,5 +32,5 @@ setup(
         "console_scripts": [
             "ambari-agent = ambari_agent.main:main",
         ],
-    }
+    }, requires=['jinja2']
 )

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/packages/windows.xml
----------------------------------------------------------------------
diff --git a/ambari-agent/src/packages/windows.xml b/ambari-agent/src/packages/windows.xml
new file mode 100644
index 0000000..b24c73e
--- /dev/null
+++ b/ambari-agent/src/packages/windows.xml
@@ -0,0 +1,82 @@
+<?xml version="1.0"?>
+<!--
+   Licensed to the Apache Software Foundation (ASF) under one or more
+   contributor license agreements.  See the NOTICE file distributed with
+   this work for additional information regarding copyright ownership.
+   The ASF licenses this file to You under the Apache License, Version 2.0
+   (the "License"); you may not use this file except in compliance with
+   the License.  You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+-->
+<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1"
+          xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.1 http://maven.apache.org/xsd/assembly-1.1.1.xsd">
+  <!--This 'all' id is not appended to the produced bundle because we do this:
+    http://maven.apache.org/plugins/maven-assembly-plugin/faq.html#required-classifiers
+  -->
+  <id>windows-dist</id>
+  <formats>
+    <format>dir</format>
+  </formats>
+  <includeBaseDirectory>false</includeBaseDirectory>
+  <fileSets>
+    <fileSet>
+      <directory>src/main/python/ambari_agent</directory>
+      <outputDirectory>/sbin/ambari_agent</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>src/main/python/resource_management</directory>
+      <outputDirectory>/sbin/resource_management</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/../ambari-common/src/main/python/ambari_commons</directory>
+      <outputDirectory>/sbin/ambari_commons</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/../ambari-common/src/main/python/jinja2/jinja2</directory>
+      <outputDirectory>/sbin/jinja2</outputDirectory>
+    </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/conf/windows</directory>
+      <outputDirectory>/</outputDirectory>
+      <excludes>
+        <exclude>service_wrapper.py</exclude>
+        <exclude>createservice.ps1</exclude>
+      </excludes>
+    </fileSet>
+    <fileSet>
+      <directory>${project.basedir}/conf/windows</directory>
+      <outputDirectory>/sbin</outputDirectory>
+      <includes>
+        <include>service_wrapper.py</include>
+        <include>createservice.ps1</include>
+      </includes>
+    </fileSet>
+    <fileSet>
+      <directory>${target.cache.dir}</directory>
+      <outputDirectory>/cache</outputDirectory>
+    </fileSet>
+    <!--empty directory-->
+    <fileSet>
+      <directory>./</directory>
+      <outputDirectory>/keys</outputDirectory>
+      <excludes>
+        <exclude>*/**</exclude>
+      </excludes>
+    </fileSet>
+  </fileSets>
+  <files>
+    <file>
+      <source>${project.basedir}/../version</source>
+      <outputDirectory>data</outputDirectory>
+      <filtered>true</filtered>
+    </file>
+  </files>
+</assembly>

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
index 874b4c9..897493a 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestActionQueue.py
@@ -27,7 +27,6 @@ import os, errno, time, pprint, tempfile, threading, json
 import StringIO
 import sys
 from threading import Thread
-import copy
 
 from mock.mock import patch, MagicMock, call
 from ambari_agent.StackVersionsFileHandler import StackVersionsFileHandler
@@ -35,11 +34,13 @@ from ambari_agent.CustomServiceOrchestrator import CustomServiceOrchestrator
 from ambari_agent.PythonExecutor import PythonExecutor
 from ambari_agent.CommandStatusDict import CommandStatusDict
 from ambari_agent.ActualConfigHandler import ActualConfigHandler
-from FileCache import FileCache
 
 
 class TestActionQueue(TestCase):
+
   def setUp(self):
+    out = StringIO.StringIO()
+    sys.stdout = out
     # save original open() method for later use
     self.original_open = open
 
@@ -142,19 +143,6 @@ class TestActionQueue(TestCase):
     'serviceName': u'HDFS',
     'configurations':{'global' : {}},
     'configurationTags':{'global' : { 'tag': 'v123' }},
-    'hostLevelParams':{'custom_command': 'RESTART', 'clientsToUpdateConfigs': []}
-  }
-
-  datanode_restart_command_no_clients_update = {
-    'commandType': 'EXECUTION_COMMAND',
-    'role': u'DATANODE',
-    'roleCommand': u'CUSTOM_COMMAND',
-    'commandId': '1-1',
-    'taskId': 9,
-    'clusterName': u'cc',
-    'serviceName': u'HDFS',
-    'configurations':{'global' : {}},
-    'configurationTags':{'global' : { 'tag': 'v123' }},
     'hostLevelParams':{'custom_command': 'RESTART'}
   }
 
@@ -167,49 +155,6 @@ class TestActionQueue(TestCase):
     'hostLevelParams': {}
   }
 
-  background_command = {
-    'commandType': 'BACKGROUND_EXECUTION_COMMAND',
-    'role': 'NAMENODE',
-    'roleCommand': 'CUSTOM_COMMAND',
-    'commandId': '1-1',
-    'taskId': 19,
-    'clusterName': 'c1',
-    'serviceName': 'HDFS',
-    'configurations':{'global' : {}},
-    'configurationTags':{'global' : { 'tag': 'v123' }},
-    'hostLevelParams':{'custom_command': 'REBALANCE_HDFS'},
-    'commandParams' :  {
-      'script_type' : 'PYTHON',
-      'script' : 'script.py',
-      'command_timeout' : '600',
-      'jdk_location' : '.',
-      'service_package_folder' : '.'
-      }
-  }
-  cancel_background_command = {
-    'commandType': 'EXECUTION_COMMAND',
-    'role': 'NAMENODE',
-    'roleCommand': 'ACTIONEXECUTE',
-    'commandId': '1-1',
-    'taskId': 20,
-    'clusterName': 'c1',
-    'serviceName': 'HDFS',
-    'configurations':{'global' : {}},
-    'configurationTags':{'global' : {}},
-    'hostLevelParams':{},
-    'commandParams' :  {
-      'script_type' : 'PYTHON',
-      'script' : 'cancel_background_task.py',
-      'before_system_hook_function' : 'fetch_bg_pid_by_taskid',
-      'jdk_location' : '.',
-      'command_timeout' : '600',
-      'service_package_folder' : '.',
-      'cancel_policy': 'SIGKILL',
-      'cancel_task_id': "19",
-      }
-  }
-
-
   @patch.object(ActionQueue, "process_command")
   @patch.object(Queue, "get")
   @patch.object(CustomServiceOrchestrator, "__init__")
@@ -233,9 +178,7 @@ class TestActionQueue(TestCase):
   def test_process_command(self, execute_status_command_mock,
                            execute_command_mock, print_exc_mock):
     dummy_controller = MagicMock()
-    config = AmbariConfig()
-    config.set('agent', 'tolerate_download_failures', "true")
-    actionQueue = ActionQueue(config, dummy_controller)
+    actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
     execution_command = {
       'commandType' : ActionQueue.EXECUTION_COMMAND,
     }
@@ -300,7 +243,7 @@ class TestActionQueue(TestCase):
         return self.original_open(file, mode)
     open_mock.side_effect = open_side_effect
 
-    config = AmbariConfig()
+    config = AmbariConfig().getConfig()
     tempdir = tempfile.gettempdir()
     config.set('agent', 'prefix', tempdir)
     config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
@@ -486,48 +429,6 @@ class TestActionQueue(TestCase):
     self.assertEqual(len(report['reports']), 1)
     self.assertEqual(expected, report['reports'][0])
 
-  @patch.object(ActualConfigHandler, "write_client_components")
-  @patch.object(CustomServiceOrchestrator, "runCommand")
-  @patch("CommandStatusDict.CommandStatusDict")
-  @patch.object(ActionQueue, "status_update_callback")
-  def test_store_configuration_tags_no_clients(self, status_update_callback_mock,
-                                    command_status_dict_mock,
-                                    cso_runCommand_mock, write_client_components_mock):
-    custom_service_orchestrator_execution_result_dict = {
-      'stdout': 'out',
-      'stderr': 'stderr',
-      'structuredOut' : '',
-      'exitcode' : 0
-    }
-    cso_runCommand_mock.return_value = custom_service_orchestrator_execution_result_dict
-
-    config = AmbariConfig().getConfig()
-    tempdir = tempfile.gettempdir()
-    config.set('agent', 'prefix', tempdir)
-    config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
-    config.set('agent', 'tolerate_download_failures', "true")
-    dummy_controller = MagicMock()
-    actionQueue = ActionQueue(config, dummy_controller)
-    actionQueue.execute_command(self.datanode_restart_command_no_clients_update)
-    report = actionQueue.result()
-    expected = {'status': 'COMPLETED',
-                'configurationTags': {'global': {'tag': 'v123'}},
-                'stderr': 'stderr',
-                'stdout': 'out',
-                'clusterName': u'cc',
-                'structuredOut': '""',
-                'roleCommand': u'CUSTOM_COMMAND',
-                'serviceName': u'HDFS',
-                'role': u'DATANODE',
-                'actionId': '1-1',
-                'taskId': 9,
-                'customCommand': 'RESTART',
-                'exitCode': 0}
-    # Agent caches configurationTags if custom_command RESTART completed
-    self.assertEqual(len(report['reports']), 1)
-    self.assertEqual(expected, report['reports'][0])
-    self.assertFalse(write_client_components_mock.called)
-
   @patch.object(ActionQueue, "status_update_callback")
   @patch.object(StackVersionsFileHandler, "read_stack_version")
   @patch.object(CustomServiceOrchestrator, "requestComponentStatus")
@@ -584,163 +485,3 @@ class TestActionQueue(TestCase):
     self.assertTrue(requestComponentStatus_mock.called)
     self.assertEqual(len(report['componentStatus']), 1)
     self.assertTrue(report['componentStatus'][0].has_key('alerts'))
-
-  @patch.object(ActionQueue, "process_command")
-  @patch.object(Queue, "get")
-  @patch.object(CustomServiceOrchestrator, "__init__")
-  def test_reset_queue(self, CustomServiceOrchestrator_mock,
-                                get_mock, process_command_mock):
-    CustomServiceOrchestrator_mock.return_value = None
-    dummy_controller = MagicMock()
-    config = MagicMock()
-    actionQueue = ActionQueue(config, dummy_controller)
-    actionQueue.start()
-    actionQueue.put([self.datanode_install_command, self.hbase_install_command])
-    self.assertEqual(2, actionQueue.commandQueue.qsize())
-    actionQueue.reset()
-    self.assertTrue(actionQueue.commandQueue.empty())
-    time.sleep(0.1)
-    actionQueue.stop()
-    actionQueue.join()
-    self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')
-
-  @patch.object(ActionQueue, "process_command")
-  @patch.object(Queue, "get")
-  @patch.object(CustomServiceOrchestrator, "__init__")
-  def test_cancel(self, CustomServiceOrchestrator_mock,
-                       get_mock, process_command_mock):
-    CustomServiceOrchestrator_mock.return_value = None
-    dummy_controller = MagicMock()
-    config = MagicMock()
-    actionQueue = ActionQueue(config, dummy_controller)
-    actionQueue.start()
-    actionQueue.put([self.datanode_install_command, self.hbase_install_command])
-    self.assertEqual(2, actionQueue.commandQueue.qsize())
-    actionQueue.reset()
-    self.assertTrue(actionQueue.commandQueue.empty())
-    time.sleep(0.1)
-    actionQueue.stop()
-    actionQueue.join()
-    self.assertEqual(actionQueue.stopped(), True, 'Action queue is not stopped.')
-
-  @patch.object(StackVersionsFileHandler, "read_stack_version")
-  @patch.object(CustomServiceOrchestrator, "runCommand")
-  @patch.object(CustomServiceOrchestrator, "__init__")
-  def test_execute_background_command(self, CustomServiceOrchestrator_mock,
-                                  runCommand_mock, read_stack_version_mock
-                                  ):
-    CustomServiceOrchestrator_mock.return_value = None
-    CustomServiceOrchestrator.runCommand.return_value = {'exitcode' : 0,
-                                                         'stdout': 'out-11',
-                                                         'stderr' : 'err-13'}
-    
-    dummy_controller = MagicMock()
-    actionQueue = ActionQueue(AmbariConfig().getConfig(), dummy_controller)
-
-    execute_command = copy.deepcopy(self.background_command)
-    actionQueue.put([execute_command])
-    actionQueue.processBackgroundQueueSafeEmpty();
-    actionQueue.processStatusCommandQueueSafeEmpty();
-    
-    #assert that python execturor start
-    self.assertTrue(runCommand_mock.called)
-    runningCommand = actionQueue.commandStatuses.current_state.get(execute_command['taskId'])
-    self.assertTrue(runningCommand is not None)
-    self.assertEqual(runningCommand[1]['status'], ActionQueue.IN_PROGRESS_STATUS)
-    
-    report = actionQueue.result()
-    self.assertEqual(len(report['reports']),1)
-      
-  @patch.object(CustomServiceOrchestrator, "resolve_script_path")
-  @patch.object(StackVersionsFileHandler, "read_stack_version")
-  def test_execute_python_executor(self, read_stack_version_mock, resolve_script_path_mock):
-    
-    dummy_controller = MagicMock()
-    cfg = AmbariConfig().getConfig()
-    cfg.set('agent', 'tolerate_download_failures', 'true')
-    cfg.set('agent', 'prefix', '.')
-    cfg.set('agent', 'cache_dir', 'background_tasks')
-    
-    actionQueue = ActionQueue(cfg, dummy_controller)
-    patch_output_file(actionQueue.customServiceOrchestrator.python_executor)
-    actionQueue.customServiceOrchestrator.dump_command_to_json = MagicMock()
-   
-    result = {}
-    lock = threading.RLock()
-    complete_done = threading.Condition(lock)
-    
-    def command_complete_w(process_condenced_result, handle):
-      with lock:
-        result['command_complete'] = {'condenced_result' : copy.copy(process_condenced_result), 
-                                      'handle' : copy.copy(handle),
-                                      'command_status' : actionQueue.commandStatuses.get_command_status(handle.command['taskId'])
-                                      }
-        complete_done.notifyAll()
-    
-    actionQueue.on_background_command_complete_callback = wraped(actionQueue.on_background_command_complete_callback,None, command_complete_w)
-    actionQueue.put([self.background_command])
-    actionQueue.processBackgroundQueueSafeEmpty();
-    actionQueue.processStatusCommandQueueSafeEmpty();
-    
-    with lock:
-      complete_done.wait(.1)
-      
-      finished_status = result['command_complete']['command_status']
-      self.assertEqual(finished_status['status'], ActionQueue.COMPLETED_STATUS)
-      self.assertEqual(finished_status['stdout'], 'process_out')
-      self.assertEqual(finished_status['stderr'], 'process_err')
-      self.assertEqual(finished_status['exitCode'], 0)
-      
-    
-    runningCommand = actionQueue.commandStatuses.current_state.get(self.background_command['taskId'])
-    self.assertTrue(runningCommand is not None)
-    
-    report = actionQueue.result()
-    self.assertEqual(len(report['reports']),1)
-    self.assertEqual(report['reports'][0]['stdout'],'process_out')
-#    self.assertEqual(report['reports'][0]['structuredOut'],'{"a": "b."}')
-    
-    
-  
-  cancel_background_command = {
-    "commandType":"CANCEL_COMMAND",
-    "role":"AMBARI_SERVER_ACTION",
-    "roleCommand":"ABORT",
-    "commandId":"2--1",
-    "taskId":20,
-    "clusterName":"c1",
-    "serviceName":"",
-    "hostname":"c6401",
-    "roleParams":{
-      "cancelTaskIdTargets":"13,14"
-    },
-  }
-
-def patch_output_file(pythonExecutor):
-  def windows_py(command, tmpout, tmperr):
-    proc = MagicMock()
-    proc.pid = 33
-    proc.returncode = 0
-    with tmpout:
-      tmpout.write('process_out')
-    with tmperr:
-      tmperr.write('process_err')
-    return proc
-  def open_subporcess_files_win(fout, ferr, f):
-    return MagicMock(), MagicMock()
-  def read_result_from_files(out_path, err_path, structured_out_path):
-    return 'process_out', 'process_err', '{"a": "b."}'
-  pythonExecutor.launch_python_subprocess = windows_py
-  pythonExecutor.open_subporcess_files = open_subporcess_files_win
-  pythonExecutor.read_result_from_files = read_result_from_files
-    
-def wraped(func, before = None, after = None):
-    def wrapper(*args, **kwargs):
-      if(before is not None):
-        before(*args, **kwargs)
-      ret =  func(*args, **kwargs)
-      if(after is not None):
-        after(*args, **kwargs)
-      return ret
-    return wrapper   
-  

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestActualConfigHandler.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestActualConfigHandler.py b/ambari-agent/src/test/python/ambari_agent/TestActualConfigHandler.py
index c659a03..ca56350 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestActualConfigHandler.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestActualConfigHandler.py
@@ -34,8 +34,8 @@ class TestActualConfigHandler(TestCase):
   def setUp(self):
     LiveStatus.SERVICES = [
       "HDFS", "MAPREDUCE", "GANGLIA", "HBASE",
-      "NAGIOS", "ZOOKEEPER", "OOZIE",
-      "KERBEROS", "TEMPLETON", "HIVE",
+      "NAGIOS", "ZOOKEEPER", "OOZIE", "HCATALOG",
+      "KERBEROS", "TEMPLETON", "HIVE", "WEBHCAT",
       "YARN", "MAPREDUCE2", "FLUME", "TEZ",
       "FALCON", "STORM"
     ]
@@ -108,7 +108,7 @@ class TestActualConfigHandler(TestCase):
        "componentName" : "HIVE_METASTORE"},
       {"serviceName" : "HIVE",
        "componentName" : "MYSQL_SERVER"},
-      {"serviceName" : "HIVE",
+      {"serviceName" : "WEBHCAT",
        "componentName" : "WEBHCAT_SERVER"},
       {"serviceName" : "YARN",
        "componentName" : "RESOURCEMANAGER"},
@@ -193,7 +193,6 @@ class TestActualConfigHandler(TestCase):
 
     tags1 = { "global": "version1", "core-site": "version2" }
     tags2 = { "global": "version33", "core-site": "version33" }
-    clientsToUpdateConfigs1 = ["*"]
     handler = ActualConfigHandler(config, {})
     handler.write_actual_component('HDFS_CLIENT', tags1)
     handler.write_actual_component('HBASE_CLIENT', tags1)
@@ -202,7 +201,7 @@ class TestActualConfigHandler(TestCase):
     handler.write_actual_component('DATANODE', tags2)
     self.assertEquals(tags2, handler.read_actual_component('DATANODE'))
     self.assertEquals(tags1, handler.read_actual_component('HDFS_CLIENT'))
-    handler.write_client_components('HDFS', tags2, clientsToUpdateConfigs1)
+    handler.write_client_components('HDFS', tags2)
     self.assertEquals(tags2, handler.read_actual_component('HDFS_CLIENT'))
     self.assertEquals(tags1, handler.read_actual_component('HBASE_CLIENT'))
 
@@ -219,37 +218,17 @@ class TestActualConfigHandler(TestCase):
     tags0 = {"global": "version0", "core-site": "version0"}
     tags1 = {"global": "version1", "core-site": "version2"}
     tags2 = {"global": "version33", "core-site": "version33"}
-    clientsToUpdateConfigs1 = ["HDFS_CLIENT","HBASE_CLIENT"]
     configTags = {'HDFS_CLIENT': tags0, 'HBASE_CLIENT': tags1}
     handler = ActualConfigHandler(config, configTags)
     self.assertEquals(tags0, handler.read_actual_component('HDFS_CLIENT'))
     self.assertEquals(tags1, handler.read_actual_component('HBASE_CLIENT'))
-    handler.write_client_components('HDFS', tags2, clientsToUpdateConfigs1)
+    handler.write_client_components('HDFS', tags2)
     self.assertEquals(tags2, handler.read_actual_component('HDFS_CLIENT'))
     self.assertEquals(tags1, handler.read_actual_component('HBASE_CLIENT'))
     self.assertTrue(write_file_mock.called)
     self.assertEqual(1, write_file_mock.call_count)
 
   @patch.object(ActualConfigHandler, "write_file")
-  def test_write_empty_client_components(self, write_file_mock):
-    config = AmbariConfig().getConfig()
-    tmpdir = tempfile.gettempdir()
-    config.set('agent', 'prefix', tmpdir)
-
-    tags0 = {"global": "version0", "core-site": "version0"}
-    tags1 = {"global": "version1", "core-site": "version2"}
-    tags2 = {"global": "version33", "core-site": "version33"}
-    clientsToUpdateConfigs1 = []
-    configTags = {'HDFS_CLIENT': tags0, 'HBASE_CLIENT': tags1}
-    handler = ActualConfigHandler(config, configTags)
-    self.assertEquals(tags0, handler.read_actual_component('HDFS_CLIENT'))
-    self.assertEquals(tags1, handler.read_actual_component('HBASE_CLIENT'))
-    handler.write_client_components('HDFS', tags2, clientsToUpdateConfigs1)
-    self.assertEquals(tags0, handler.read_actual_component('HDFS_CLIENT'))
-    self.assertEquals(tags1, handler.read_actual_component('HBASE_CLIENT'))
-    self.assertFalse(write_file_mock.called)
-
-  @patch.object(ActualConfigHandler, "write_file")
   @patch.object(ActualConfigHandler, "read_file")
   def test_read_actual_component_inmemory(self, read_file_mock, write_file_mock):
     config = AmbariConfig().getConfig()

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
index 8e01707..5216447 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCertGeneration.py
@@ -29,20 +29,20 @@ from ambari_agent import AmbariConfig
 class TestCertGeneration(TestCase):
   def setUp(self):
     self.tmpdir = tempfile.mkdtemp()
-    config = AmbariConfig.AmbariConfig()
-    #config.add_section('server')
+    config = ConfigParser.RawConfigParser()
+    config.add_section('server')
     config.set('server', 'hostname', 'example.com')
     config.set('server', 'url_port', '777')
-    #config.add_section('security')
+    config.add_section('security')
     config.set('security', 'keysdir', self.tmpdir)
     config.set('security', 'server_crt', 'ca.crt')
     self.certMan = CertificateManager(config)
-
+    
   def test_generation(self):
     self.certMan.genAgentCrtReq()
     self.assertTrue(os.path.exists(self.certMan.getAgentKeyName()))
     self.assertTrue(os.path.exists(self.certMan.getAgentCrtReqName()))
   def tearDown(self):
     shutil.rmtree(self.tmpdir)
-
+    
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestController.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestController.py b/ambari-agent/src/test/python/ambari_agent/TestController.py
index 7ef64cc..dd92e06 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestController.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestController.py
@@ -30,7 +30,7 @@ from threading import Event
 import json
 
 with patch("platform.linux_distribution", return_value = ('Suse','11','Final')):
-  from ambari_agent import Controller, ActionQueue, Register
+  from ambari_agent import Controller, ActionQueue
   from ambari_agent import hostname
   from ambari_agent.Controller import AGENT_AUTO_RESTART_EXIT_CODE
   from ambari_commons import OSCheck
@@ -53,8 +53,7 @@ class TestController(unittest.TestCase):
 
 
     config = MagicMock()
-    #config.get.return_value = "something"
-    config.get.return_value = "5"
+    config.get.return_value = "something"
 
     self.controller = Controller.Controller(config)
     self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS = 0.1
@@ -236,8 +235,6 @@ class TestController(unittest.TestCase):
     self.controller.registerWithServer = registerWithServer
     heartbeatWithServer = MagicMock(name="heartbeatWithServer")
     self.controller.heartbeatWithServer = heartbeatWithServer
-    actionQueue = MagicMock(name="actionQueue")
-    self.controller.actionQueue = actionQueue
 
     Controller.Controller.__sendRequest__ = MagicMock(side_effect=Exception())
 
@@ -247,9 +244,9 @@ class TestController(unittest.TestCase):
     heartbeatWithServer.assert_called_once_with()
 
     self.controller.registerWithServer =\
-      Controller.Controller.registerWithServer
+    Controller.Controller.registerWithServer
     self.controller.heartbeatWithServer =\
-      Controller.Controller.registerWithServer
+    Controller.Controller.registerWithServer
 
   @patch("time.sleep")
   def test_registerAndHeartbeat(self, sleepMock):
@@ -259,8 +256,6 @@ class TestController(unittest.TestCase):
     self.controller.registerWithServer = registerWithServer
     heartbeatWithServer = MagicMock(name="heartbeatWithServer")
     self.controller.heartbeatWithServer = heartbeatWithServer
-    actionQueue = MagicMock(name="actionQueue")
-    self.controller.actionQueue = actionQueue
 
     listener1 = MagicMock()
     listener2 = MagicMock()
@@ -286,8 +281,6 @@ class TestController(unittest.TestCase):
     self.controller.registerWithServer = registerWithServer
     heartbeatWithServer = MagicMock(name="heartbeatWithServer")
     self.controller.heartbeatWithServer = heartbeatWithServer
-    actionQueue = MagicMock(name="actionQueue")
-    self.controller.actionQueue = actionQueue
 
     self.controller.isRegistered = True
     self.controller.registerAndHeartbeat()
@@ -300,33 +293,6 @@ class TestController(unittest.TestCase):
       Controller.Controller.registerWithServer
 
 
-  @patch("time.sleep")
-  @patch.object(Controller.Controller, "sendRequest")
-  def test_registerWithIOErrors(self, sendRequestMock, sleepMock):
-    # Check that server continues to heartbeat after connection errors
-    registerMock = MagicMock(name="Register")
-    registerMock.build.return_value = {}
-    actionQueue = MagicMock()
-    actionQueue.isIdle.return_value = True
-    self.controller.actionQueue = actionQueue
-    self.controller.register = registerMock
-    self.controller.responseId = 1
-    self.controller.TEST_IOERROR_COUNTER = 1
-    self.controller.isRegistered = False
-    def util_throw_IOErrors(*args, **kwargs):
-      """
-      Throws IOErrors 10 times and then stops heartbeats/registrations
-      """
-      if self.controller.TEST_IOERROR_COUNTER == 10:
-        self.controller.isRegistered = True
-      self.controller.TEST_IOERROR_COUNTER += 1
-      raise IOError("Sample error")
-    actionQueue.isIdle.return_value = False
-    sendRequestMock.side_effect = util_throw_IOErrors
-    self.controller.registerWithServer()
-    self.assertTrue(sendRequestMock.call_count > 5)
-
-
   @patch("os._exit")
   def test_restartAgent(self, os_exit_mock):
 
@@ -358,22 +324,18 @@ class TestController(unittest.TestCase):
       {'Content-Type': 'application/json'})
 
     conMock.request.return_value = '{invalid_object}'
-
-    try:
-      self.controller.sendRequest(url, data)
-      self.fail("Should throw exception!")
-    except IOError, e: # Expected
-      self.assertEquals('Response parsing failed! Request data: ' + data +
-                        '; Response: {invalid_object}', str(e))
+    actual = self.controller.sendRequest(url, data)
+    expected = {'exitstatus': 1, 'log': ('Response parsing failed! Request data: ' + data
+                                         + '; Response: {invalid_object}')}
+    self.assertEqual(actual, expected)
 
     exceptionMessage = "Connection Refused"
     conMock.request.side_effect = Exception(exceptionMessage)
-    try:
-      self.controller.sendRequest(url, data)
-      self.fail("Should throw exception!")
-    except IOError, e: # Expected
-      self.assertEquals('Request to ' + url + ' failed due to ' +
-                        exceptionMessage, str(e))
+    actual = self.controller.sendRequest(url, data)
+    expected = {'exitstatus': 1, 'log': 'Request to ' + url + ' failed due to ' + exceptionMessage}
+
+    self.assertEqual(actual, expected)
+
 
 
   @patch.object(threading._Event, "wait")
@@ -514,27 +476,6 @@ class TestController(unittest.TestCase):
     sleepMock.assert_called_with(
       self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS)
 
-    # Check that server continues to heartbeat after connection errors
-    self.controller.responseId = 1
-    self.controller.TEST_IOERROR_COUNTER = 1
-    sendRequest.reset()
-    def util_throw_IOErrors(*args, **kwargs):
-      """
-      Throws IOErrors 100 times and then stops heartbeats/registrations
-      """
-      if self.controller.TEST_IOERROR_COUNTER == 10:
-        self.controller.DEBUG_STOP_HEARTBEATING = True
-      self.controller.TEST_IOERROR_COUNTER += 1
-      raise IOError("Sample error")
-    self.controller.DEBUG_STOP_HEARTBEATING = False
-    actionQueue.isIdle.return_value = False
-    sendRequest.side_effect = util_throw_IOErrors
-    self.controller.heartbeatWithServer()
-    self.assertTrue(sendRequest.call_count > 5)
-
-    sleepMock.assert_called_with(
-      self.controller.netutil.MINIMUM_INTERVAL_BETWEEN_HEARTBEATS)
-
     sys.stdout = sys.__stdout__
     self.controller.sendRequest = Controller.Controller.sendRequest
     self.controller.sendRequest = Controller.Controller.addToQueue

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
index 6a61c0d..23b9bed 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestCustomServiceOrchestrator.py
@@ -18,11 +18,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 '''
 import ConfigParser
-from multiprocessing.pool import ThreadPool
 import os
 
 import pprint
-import shell
 
 from unittest import TestCase
 import threading
@@ -39,8 +37,6 @@ import sys
 from AgentException import AgentException
 from FileCache import FileCache
 from LiveStatus import LiveStatus
-from BackgroundCommandExecutionHandle import BackgroundCommandExecutionHandle
-from ambari_agent.ActionQueue import ActionQueue
 
 
 class TestCustomServiceOrchestrator(TestCase):
@@ -51,11 +47,9 @@ class TestCustomServiceOrchestrator(TestCase):
     sys.stdout = out
     # generate sample config
     tmpdir = tempfile.gettempdir()
-    exec_tmp_dir = os.path.join(tmpdir, 'tmp')
     self.config = ConfigParser.RawConfigParser()
     self.config.add_section('agent')
     self.config.set('agent', 'prefix', tmpdir)
-    self.config.set('agent', 'tmp_dir', exec_tmp_dir)
     self.config.set('agent', 'cache_dir', "/cachedir")
     self.config.add_section('python')
     self.config.set('python', 'custom_actions_dir', tmpdir)
@@ -189,8 +183,6 @@ class TestCustomServiceOrchestrator(TestCase):
        '/hooks_dir/prefix-command')
     dummy_controller = MagicMock()
     orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
-    unix_process_id = 111
-    orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
     get_hook_base_dir_mock.return_value = "/hooks/"
     # normal run case
     run_file_mock.return_value = {
@@ -214,9 +206,9 @@ class TestCustomServiceOrchestrator(TestCase):
     ret = orchestrator.runCommand(command, "out.txt", "err.txt",
               forced_command_name=CustomServiceOrchestrator.COMMAND_NAME_STATUS)
     ## Check that override_output_files was true only during first call
-    self.assertEquals(run_file_mock.call_args_list[0][0][10], True)
-    self.assertEquals(run_file_mock.call_args_list[1][0][10], False)
-    self.assertEquals(run_file_mock.call_args_list[2][0][10], False)
+    self.assertEquals(run_file_mock.call_args_list[0][0][7], True)
+    self.assertEquals(run_file_mock.call_args_list[1][0][7], False)
+    self.assertEquals(run_file_mock.call_args_list[2][0][7], False)
     ## Check that forced_command_name was taken into account
     self.assertEqual(run_file_mock.call_args_list[0][0][1][0],
                                   CustomServiceOrchestrator.COMMAND_NAME_STATUS)
@@ -235,146 +227,6 @@ class TestCustomServiceOrchestrator(TestCase):
 
     pass
 
-  @patch("shell.kill_process_with_children")
-  @patch.object(CustomServiceOrchestrator, "resolve_script_path")
-  @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
-  @patch.object(FileCache, "get_service_base_dir")
-  @patch.object(FileCache, "get_hook_base_dir")
-  @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
-  @patch.object(PythonExecutor, "run_file")
-  @patch.object(FileCache, "__init__")
-  def test_cancel_command(self, FileCache_mock,
-                      run_file_mock, dump_command_to_json_mock,
-                      get_hook_base_dir_mock, get_service_base_dir_mock,
-                      resolve_hook_script_path_mock, resolve_script_path_mock,
-                      kill_process_with_children_mock):
-    FileCache_mock.return_value = None
-    command = {
-      'role' : 'REGION_SERVER',
-      'hostLevelParams' : {
-        'stack_name' : 'HDP',
-        'stack_version' : '2.0.7',
-        'jdk_location' : 'some_location'
-      },
-      'commandParams': {
-        'script_type': 'PYTHON',
-        'script': 'scripts/hbase_regionserver.py',
-        'command_timeout': '600',
-        'service_package_folder' : 'HBASE'
-      },
-      'taskId' : '3',
-      'roleCommand': 'INSTALL'
-    }
-    get_service_base_dir_mock.return_value = "/basedir/"
-    resolve_script_path_mock.return_value = "/basedir/scriptpath"
-    resolve_hook_script_path_mock.return_value = \
-      ('/hooks_dir/prefix-command/scripts/hook.py',
-       '/hooks_dir/prefix-command')
-    dummy_controller = MagicMock()
-    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
-    unix_process_id = 111
-    orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
-    get_hook_base_dir_mock.return_value = "/hooks/"
-    run_file_mock_return_value = {
-      'stdout' : 'killed',
-      'stderr' : 'killed',
-      'exitcode': 1,
-      }
-    def side_effect(*args, **kwargs):
-      time.sleep(0.2)
-      return run_file_mock_return_value
-    run_file_mock.side_effect = side_effect
-
-    _, out = tempfile.mkstemp()
-    _, err = tempfile.mkstemp()
-    pool = ThreadPool(processes=1)
-    async_result = pool.apply_async(orchestrator.runCommand, (command, out, err))
-
-    time.sleep(0.1)
-    orchestrator.cancel_command(command['taskId'], 'reason')
-
-    ret = async_result.get()
-
-    self.assertEqual(ret['exitcode'], 1)
-    self.assertEquals(ret['stdout'], 'killed\nCommand aborted. reason')
-    self.assertEquals(ret['stderr'], 'killed\nCommand aborted. reason')
-
-    self.assertTrue(kill_process_with_children_mock.called)
-    self.assertFalse(command['taskId'] in orchestrator.commands_in_progress.keys())
-    self.assertTrue(os.path.exists(out))
-    self.assertTrue(os.path.exists(err))
-    os.remove(out)
-    os.remove(err)
-    
-  from ambari_agent.StackVersionsFileHandler import StackVersionsFileHandler
-    
-  @patch("shell.kill_process_with_children")
-  @patch.object(FileCache, "__init__")
-  @patch.object(CustomServiceOrchestrator, "resolve_script_path")
-  @patch.object(CustomServiceOrchestrator, "resolve_hook_script_path")
-  @patch.object(StackVersionsFileHandler, "read_stack_version")
-  def test_cancel_backgound_command(self, read_stack_version_mock, resolve_hook_script_path_mock, resolve_script_path_mock, FileCache_mock,  
-                                      kill_process_with_children_mock):
-    FileCache_mock.return_value = None
-    FileCache_mock.cache_dir = MagicMock()
-    resolve_hook_script_path_mock.return_value = None
-#     shell.kill_process_with_children = MagicMock()
-    dummy_controller = MagicMock()
-    cfg = AmbariConfig().getConfig()
-    cfg.set('agent', 'tolerate_download_failures', 'true')
-    cfg.set('agent', 'prefix', '.')
-    cfg.set('agent', 'cache_dir', 'background_tasks')
-     
-    actionQueue = ActionQueue(cfg, dummy_controller)
-    
-    dummy_controller.actionQueue = actionQueue
-    orchestrator = CustomServiceOrchestrator(cfg, dummy_controller)
-    orchestrator.file_cache = MagicMock()
-    def f (a, b):
-      return ""
-    orchestrator.file_cache.get_service_base_dir = f
-    actionQueue.customServiceOrchestrator = orchestrator
-    
-    import TestActionQueue
-    import copy
-    
-    TestActionQueue.patch_output_file(orchestrator.python_executor)
-    orchestrator.python_executor.prepare_process_result = MagicMock()
-    orchestrator.dump_command_to_json = MagicMock()
- 
-    lock = threading.RLock()
-    complete_done = threading.Condition(lock)
-    
-    complete_was_called = {}
-    def command_complete_w(process_condenced_result, handle):
-      with lock:
-        complete_was_called['visited']= ''
-        complete_done.wait(3)
-     
-    actionQueue.on_background_command_complete_callback = TestActionQueue.wraped(actionQueue.on_background_command_complete_callback, command_complete_w, None) 
-    execute_command = copy.deepcopy(TestActionQueue.TestActionQueue.background_command)
-    actionQueue.put([execute_command])
-    actionQueue.processBackgroundQueueSafeEmpty()
-     
-    time.sleep(.1) 
-    
-    orchestrator.cancel_command(19,'')
-    self.assertTrue(kill_process_with_children_mock.called)
-    kill_process_with_children_mock.assert_called_with(33)
-     
-    with lock:
-      complete_done.notifyAll()
-
-    with lock:
-      self.assertTrue(complete_was_called.has_key('visited'))
-    
-    time.sleep(.1)
-     
-    runningCommand = actionQueue.commandStatuses.get_command_status(19)
-    self.assertTrue(runningCommand is not None)
-    self.assertEqual(runningCommand['status'], ActionQueue.FAILED_STATUS)
-
-
   @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
   @patch.object(PythonExecutor, "run_file")
   @patch.object(FileCache, "__init__")
@@ -398,8 +250,6 @@ class TestCustomServiceOrchestrator(TestCase):
     }
     dummy_controller = MagicMock()
     orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
-    unix_process_id = 111
-    orchestrator.commands_in_progress = {command['taskId']: unix_process_id}
     # normal run case
     run_file_mock.return_value = {
       'stdout' : 'sss',
@@ -466,40 +316,6 @@ class TestCustomServiceOrchestrator(TestCase):
     self.assertEqual(runCommand_mock.return_value, status)
 
 
-  @patch.object(CustomServiceOrchestrator, "dump_command_to_json")
-  @patch.object(FileCache, "__init__")
-  @patch.object(FileCache, "get_custom_actions_base_dir")
-  def test_runCommand_background_action(self, get_custom_actions_base_dir_mock,
-                                    FileCache_mock,
-                                    dump_command_to_json_mock):
-    FileCache_mock.return_value = None
-    get_custom_actions_base_dir_mock.return_value = "some path"
-    _, script = tempfile.mkstemp()
-    command = {
-      'role' : 'any',
-      'commandParams': {
-        'script_type': 'PYTHON',
-        'script': 'some_custom_action.py',
-        'command_timeout': '600',
-        'jdk_location' : 'some_location'
-      },
-      'taskId' : '13',
-      'roleCommand': 'ACTIONEXECUTE',
-      'commandType': 'BACKGROUND_EXECUTION_COMMAND',
-      '__handle': BackgroundCommandExecutionHandle({'taskId': '13'}, 13,
-                                                   MagicMock(), MagicMock())
-    }
-    dummy_controller = MagicMock()
-    orchestrator = CustomServiceOrchestrator(self.config, dummy_controller)
-    
-    import TestActionQueue
-    TestActionQueue.patch_output_file(orchestrator.python_executor)
-    orchestrator.python_executor.condenseOutput = MagicMock()
-    orchestrator.dump_command_to_json = MagicMock()
-    
-    ret = orchestrator.runCommand(command, "out.txt", "err.txt")
-    self.assertEqual(ret['exitcode'], 777)
-
   def tearDown(self):
     # enable stdout
     sys.stdout = sys.__stdout__

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestDataCleaner.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestDataCleaner.py b/ambari-agent/src/test/python/ambari_agent/TestDataCleaner.py
index fc79d27..d385697 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestDataCleaner.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestDataCleaner.py
@@ -22,90 +22,66 @@ limitations under the License.
 import unittest
 from mock.mock import patch, MagicMock, call, Mock
 from ambari_agent import DataCleaner
-import AmbariConfig
 
 
 class TestDataCleaner(unittest.TestCase):
 
   def setUp(self):
     self.test_dir = [('/test_path', [],
-                      ['errors-12.txt', 'output-12.txt', 'site-12.pp', 'site-13.pp', 'site-15.pp',
-                       'structured-out-13.json', 'command-13.json', 'version'])]
+                      ['errors-12.txt','output-12.txt','site-12.pp','site-13.pp','site-15.pp','version'])]
     self.config = MagicMock()
-    self.config.get.side_effect = [2592000, (3600 + 1), 10000, "/test_path"]
+    self.config.get.side_effect = [2592000,3600 + 1,"/test_path"]
     DataCleaner.logger = MagicMock()
 
   def test_init_success(self):
     config = MagicMock()
-    config.get.side_effect = [2592000, (3600 + 1), 10000, "/test_path"]
+    config.get.return_value = 2592000
     DataCleaner.logger.reset_mock()
     cleaner = DataCleaner.DataCleaner(config)
     self.assertFalse(DataCleaner.logger.warn.called)
 
-  def test_config(self):
-    """
-    Verify that if the config does not have a property, default values are used.
-    """
-    DataCleaner.logger.reset_mock()
-    config = AmbariConfig.AmbariConfig()
-    config.remove_option('agent', 'data_cleanup_max_age')
-    config.remove_option('agent', 'data_cleanup_interval')
-    config.remove_option('agent', 'data_cleanup_max_size_MB')
-    cleaner = DataCleaner.DataCleaner(config)
-
-    self.assertEqual(cleaner.file_max_age, 86400)
-    self.assertEqual(cleaner.cleanup_interval, 3600)
-    self.assertEqual(cleaner.cleanup_max_size_MB, 10000)
 
   def test_init_warn(self):
     config = MagicMock()
-    config.get.side_effect = [1, (3600 - 1), (10000 + 1), "/test_path"]
+    config.get.return_value = 10
     DataCleaner.logger.reset_mock()
     cleaner = DataCleaner.DataCleaner(config)
     self.assertTrue(DataCleaner.logger.warn.called)
-    self.assertTrue(cleaner.file_max_age == 86400)
-    self.assertTrue(cleaner.cleanup_interval == 3600)
-    self.assertTrue(cleaner.cleanup_max_size_MB == 10000)
+    self.assertTrue(cleaner.file_max_age == 3600)
 
   @patch('os.walk')
   @patch('time.time')
   @patch('os.path.getmtime')
   @patch('os.remove')
-  @patch('os.path.getsize')
-  def test_cleanup_success(self, sizeMock, remMock, mtimeMock, timeMock, walkMock):
+  def test_cleanup_success(self,remMock,mtimeMock,timeMock,walkMock):
     self.config.reset_mock()
     DataCleaner.logger.reset_mock()
 
     walkMock.return_value = iter(self.test_dir)
     timeMock.return_value = 2592000 + 2
-    mtimeMock.side_effect = [1, 1, 1, 2, 1, 1, 1, 1]
-    sizeMock.return_value = 100
+    mtimeMock.side_effect = [1,1,1,2,1,1]
 
     cleaner = DataCleaner.DataCleaner(self.config)
     cleaner.cleanup()
 
-    self.assertTrue(len(remMock.call_args_list) == 6)
-    remMock.assert_any_call('/test_path/errors-12.txt')
-    remMock.assert_any_call('/test_path/output-12.txt')
-    remMock.assert_any_call('/test_path/site-12.pp')
-    remMock.assert_any_call('/test_path/site-15.pp')
-    remMock.assert_any_call('/test_path/structured-out-13.json')
-    remMock.assert_any_call('/test_path/command-13.json')
+    self.assertTrue(len(remMock.call_args_list) == 4)
+    remMock.assert_any_call('/test_path/errors-12.txt');
+    remMock.assert_any_call('/test_path/output-12.txt');
+    remMock.assert_any_call('/test_path/site-12.pp');
+    remMock.assert_any_call('/test_path/site-15.pp');
     pass
 
   @patch('os.walk')
   @patch('time.time')
   @patch('os.path.getmtime')
   @patch('os.remove')
-  @patch('os.path.getsize')
-  def test_cleanup_remove_error(self, sizeMock, remMock, mtimeMock, timeMock, walkMock):
+  def test_cleanup_remove_error(self,remMock,mtimeMock,timeMock,walkMock):
     self.config.reset_mock()
     DataCleaner.logger.reset_mock()
 
     walkMock.return_value = iter(self.test_dir)
     timeMock.return_value = 2592000 + 2
-    mtimeMock.side_effect = [1, 1, 1, 2, 1, 1, 1, 1]
-    sizeMock.return_value = 100
+    mtimeMock.side_effect = [1,1,1,2,1,1]
 
     def side_effect(arg):
       if arg == '/test_path/site-15.pp':
@@ -116,7 +92,7 @@ class TestDataCleaner(unittest.TestCase):
     cleaner = DataCleaner.DataCleaner(self.config)
     cleaner.cleanup()
 
-    self.assertTrue(len(remMock.call_args_list) == 6)
+    self.assertTrue(len(remMock.call_args_list) == 4)
     self.assertTrue(DataCleaner.logger.error.call_count == 1)
     pass
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestHardware.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHardware.py b/ambari-agent/src/test/python/ambari_agent/TestHardware.py
index 410f9d8..96d1a65 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHardware.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHardware.py
@@ -196,10 +196,10 @@ lo        Link encap:Local Loopback
     self.assertEquals(result['operatingsystem'], 'some_type_of_os')
     self.assertEquals(result['osfamily'], 'redhat')
 
-    get_os_family_mock.return_value = "ubuntu"
+    get_os_family_mock.return_value = "debian"
     result = Facter().facterInfo()
     self.assertEquals(result['operatingsystem'], 'some_type_of_os')
-    self.assertEquals(result['osfamily'], 'ubuntu')
+    self.assertEquals(result['osfamily'], 'debian')
 
     get_os_family_mock.return_value = "suse"
     result = Facter().facterInfo()

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestHeartbeat.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHeartbeat.py b/ambari-agent/src/test/python/ambari_agent/TestHeartbeat.py
index bdc6c6b..4703af4 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHeartbeat.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHeartbeat.py
@@ -27,7 +27,6 @@ from mock.mock import patch, MagicMock, call
 import StringIO
 import sys
 
-
 with patch("platform.linux_distribution", return_value = ('Suse','11','Final')):
   from ambari_agent.Heartbeat import Heartbeat
   from ambari_agent.ActionQueue import ActionQueue
@@ -65,7 +64,7 @@ class TestHeartbeat(TestCase):
     self.assertEquals(result['componentStatus'] is not None, True, "Heartbeat should contain componentStatus")
     self.assertEquals(result['reports'] is not None, True, "Heartbeat should contain reports")
     self.assertEquals(result['timestamp'] >= 1353679373880L, True)
-    self.assertEquals(len(result['nodeStatus']), 3)
+    self.assertEquals(len(result['nodeStatus']), 2)
     self.assertEquals(result['nodeStatus']['cause'], "NONE")
     self.assertEquals(result['nodeStatus']['status'], "HEALTHY")
     # result may or may NOT have an agentEnv structure in it
@@ -103,10 +102,9 @@ class TestHeartbeat(TestCase):
     hb = heartbeat.build(id = 0, state_interval=1, componentsMapped=True)
     self.assertEqual(register_mock.call_args_list[0][0][1], False)
 
-  @patch.object(HostInfo, "createAlerts")
+
   @patch.object(ActionQueue, "result")
-  def test_build_long_result(self, result_mock, createAlerts_mock):
-    createAlerts_mock.return_value = []
+  def test_build_long_result(self, result_mock):
     config = AmbariConfig.AmbariConfig().getConfig()
     config.set('agent', 'prefix', 'tmp')
     config.set('agent', 'cache_dir', "/var/lib/ambari-agent/cache")
@@ -171,7 +169,6 @@ class TestHeartbeat(TestCase):
     hb['timestamp'] = 'timestamp'
     expected = {'nodeStatus':
                   {'status': 'HEALTHY',
-                   'alerts': [],
                    'cause': 'NONE'},
                 'timestamp': 'timestamp', 'hostname': 'hostname',
                 'responseId': 10, 'reports': [

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
index 08e7b47..ba827ae 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostInfo.py
@@ -32,10 +32,11 @@ with patch("platform.linux_distribution", return_value = ('redhat','11','Final')
   from ambari_agent.HostCheckReportFileHandler import HostCheckReportFileHandler
   from ambari_agent.PackagesAnalyzer import PackagesAnalyzer
   from ambari_agent.HostInfo import HostInfo
+  from ambari_agent.HostInfo import FirewallChecks
   from ambari_agent.Hardware import Hardware
   from ambari_agent.AmbariConfig import AmbariConfig
   from resource_management.core.system import System
-  from ambari_commons import OSCheck, Firewall, FirewallChecks ,OSConst
+  from ambari_commons import OSCheck
 
 @patch.object(System, "os_family", new = 'redhat')
 class TestHostInfo(TestCase):
@@ -268,7 +269,6 @@ class TestHostInfo(TestCase):
     hostInfo = HostInfo()
     dict = {}
     hostInfo.register(dict, False, False)
-    self.assertTrue(cit_mock.called)
     self.assertTrue(gir_mock.called)
     self.assertTrue(gpd_mock.called)
     self.assertTrue(aip_mock.called)
@@ -321,7 +321,6 @@ class TestHostInfo(TestCase):
     self.verifyReturnedValues(dict)
     self.assertTrue(os_umask_mock.call_count == 2)
 
-    cit_mock.reset_mock()
     hostInfo = HostInfo()
     dict = {}
     hostInfo.register(dict, False, False)
@@ -329,7 +328,6 @@ class TestHostInfo(TestCase):
     self.assertTrue(gpd_mock.called)
     self.assertTrue(aip_mock.called)
     self.assertTrue(cit_mock.called)
-    self.assertEqual(1, cit_mock.call_count)
 
     for existingPkg in ["pkg1", "pkg2"]:
       self.assertTrue(existingPkg in dict['installedPackages'])
@@ -517,31 +515,36 @@ class TestHostInfo(TestCase):
     self.assertEquals(result[0]['name'], 'config1')
     self.assertEquals(result[0]['target'], 'real_path_to_conf')
 
-  @patch.object(OSCheck, "get_os_family")
-  @patch.object(OSCheck, "get_os_type")
-  @patch.object(OSCheck, "get_os_major_version")
   @patch.object(FirewallChecks, "run_os_command")
-  def test_IpTablesRunning(self, run_os_command_mock, get_os_major_version_mock, get_os_type_mock, get_os_family_mock):
-    get_os_type_mock.return_value = ""
-    get_os_family_mock.return_value = OSConst.REDHAT_FAMILY
-    run_os_command_mock.return_value = 0, "Table: filter", ""
-    self.assertTrue(Firewall().getFirewallObject().check_iptables())
+  def test_IpTablesRunning(self, run_os_command_mock):
+    hostInfo = HostInfo()
+    for firewallType in hostInfo.getFirewallObjectTypes():
+      firewall = firewallType()
+      run_os_command_mock.return_value = firewall.get_running_result()
+      self.assertTrue(firewall.check_iptables())
+
+  @patch("subprocess.Popen")
+  def test_run_os_command_exception(self, popen_mock):
+    def base_test():
+       return "base test"
 
+    def sub_test():
+      return "output 1", "error 1"
+
+    base_test.communicate = sub_test
+    base_test.returncode = 0
 
-  @patch.object(HostInfo, "osdiskAvailableSpace")
-  def test_createAlerts(self, osdiskAvailableSpace_mock):
     hostInfo = HostInfo()
-    osdiskAvailableSpace_mock.return_value = {
-      'size': '100',
-      'used': '50',
-      'available': '50',
-      'percent': '50%',
-      'mountpoint': '/testmount',
-      'type': 'ext4',
-      'device': 'device'}
-    result = hostInfo.createAlerts([])
-    self.assertEquals(1, len(result))
+    for firewallType in hostInfo.getFirewallObjectTypes():
+      firewall = firewallType()
+
+      popen_mock.side_effect = None
+      popen_mock.return_value = base_test
+      self.assertTrue(firewall.check_iptables())
 
+      popen_mock.side_effect = OSError('File not found')
+      popen_mock.return_value = None
+      self.assertFalse(firewall.check_iptables())
 
   @patch.object(socket, "getfqdn")
   @patch.object(socket, "gethostbyname")
@@ -567,16 +570,13 @@ class TestHostInfo(TestCase):
 
     self.assertFalse(hostInfo.checkReverseLookup())
 
-
-  @patch.object(OSCheck, "get_os_family")
-  @patch.object(OSCheck, "get_os_type")
-  @patch.object(OSCheck, "get_os_major_version")
   @patch.object(FirewallChecks, "run_os_command")
-  def test_IpTablesStopped(self, run_os_command_mock, get_os_major_version_mock, get_os_type_mock, get_os_family_mock):
-    get_os_type_mock.return_value = ""
-    get_os_family_mock.return_value = OSConst.REDHAT_FAMILY
-    run_os_command_mock.return_value = 3, "", ""
-    self.assertFalse(Firewall().getFirewallObject().check_iptables())
+  def test_IpTablesStopped(self, run_os_command_mock):
+    hostInfo = HostInfo()
+    for firewallType in hostInfo.getFirewallObjectTypes():
+      firewall = firewallType()
+      run_os_command_mock.return_value = firewall.get_stopped_result()
+      self.assertFalse(firewall.check_iptables())
 
   @patch("os.path.isfile")
   @patch('__builtin__.open')

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestHostname.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestHostname.py b/ambari-agent/src/test/python/ambari_agent/TestHostname.py
index 7d1f3c6..039ffd4 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestHostname.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestHostname.py
@@ -21,7 +21,7 @@ limitations under the License.
 from unittest import TestCase
 import unittest
 import ambari_agent.hostname as hostname
-from ambari_agent.AmbariConfig import AmbariConfig
+import ambari_agent.AmbariConfig as AmbariConfig
 import socket
 import tempfile
 import shutil
@@ -33,8 +33,7 @@ class TestHostname(TestCase):
   def test_hostname(self):
     hostname.cached_hostname = None
     hostname.cached_public_hostname = None
-    config = AmbariConfig()
-    self.assertEquals(hostname.hostname(config), socket.getfqdn(),
+    self.assertEquals(hostname.hostname(), socket.getfqdn().lower(),
                       "hostname should equal the socket-based hostname")
     pass
 
@@ -47,14 +46,14 @@ class TestHostname(TestCase):
     os.chmod(tmpname, os.stat(tmpname).st_mode | stat.S_IXUSR)
 
     tmpfile = file(tmpname, "w+")
-    config = AmbariConfig()
+    config = AmbariConfig.config
     try:
       tmpfile.write("#!/bin/sh\n\necho 'test.example.com'")
       tmpfile.close()
 
       config.set('agent', 'hostname_script', tmpname)
 
-      self.assertEquals(hostname.hostname(config), 'test.example.com', "expected hostname 'test.example.com'")
+      self.assertEquals(hostname.hostname(), 'test.example.com', "expected hostname 'test.example.com'")
     finally:
       os.remove(tmpname)
       config.remove_option('agent', 'hostname_script')
@@ -67,17 +66,17 @@ class TestHostname(TestCase):
     tmpname = fd[1]
     os.close(fd[0])
     os.chmod(tmpname, os.stat(tmpname).st_mode | stat.S_IXUSR)
-
+   
     tmpfile = file(tmpname, "w+")
 
-    config = AmbariConfig()
+    config = AmbariConfig.config
     try:
       tmpfile.write("#!/bin/sh\n\necho 'test.example.com'")
       tmpfile.close()
 
       config.set('agent', 'public_hostname_script', tmpname)
 
-      self.assertEquals(hostname.public_hostname(config), 'test.example.com',
+      self.assertEquals(hostname.public_hostname(), 'test.example.com', 
                         "expected hostname 'test.example.com'")
     finally:
       os.remove(tmpname)
@@ -88,10 +87,9 @@ class TestHostname(TestCase):
   def test_caching(self, getfqdn_mock):
     hostname.cached_hostname = None
     hostname.cached_public_hostname = None
-    config = AmbariConfig()
     getfqdn_mock.side_effect = ["test.example.com", "test2.example.com'"]
-    self.assertEquals(hostname.hostname(config), "test.example.com")
-    self.assertEquals(hostname.hostname(config), "test.example.com")
+    self.assertEquals(hostname.hostname(), "test.example.com")
+    self.assertEquals(hostname.hostname(), "test.example.com")
     self.assertEqual(getfqdn_mock.call_count, 1)
     pass
 
@@ -99,3 +97,5 @@ if __name__ == "__main__":
   unittest.main(verbosity=2)
 
 
+
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestLiveStatus.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestLiveStatus.py b/ambari-agent/src/test/python/ambari_agent/TestLiveStatus.py
index 9dfb47a..6937da2 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestLiveStatus.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestLiveStatus.py
@@ -36,8 +36,8 @@ class TestLiveStatus(TestCase):
     sys.stdout = out
     LiveStatus.SERVICES = [
       "HDFS", "MAPREDUCE", "GANGLIA", "HBASE",
-      "NAGIOS", "ZOOKEEPER", "OOZIE",
-      "KERBEROS", "TEMPLETON", "HIVE",
+      "NAGIOS", "ZOOKEEPER", "OOZIE", "HCATALOG",
+      "KERBEROS", "TEMPLETON", "HIVE", "WEBHCAT",
       "YARN", "MAPREDUCE2", "FLUME", "TEZ",
       "FALCON", "STORM"
     ]

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestMain.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestMain.py b/ambari-agent/src/test/python/ambari_agent/TestMain.py
index 7b1a8c8..7cf14b6 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestMain.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestMain.py
@@ -242,11 +242,11 @@ class TestMain(unittest.TestCase):
   @patch.object(DataCleaner,"start")
   @patch.object(DataCleaner,"__init__")
   @patch.object(PingPortListener,"start")
-  @patch.object(PingPortListener,"__init__")
+  @patch.object(PingPortListener,"__init__")  
   def test_main(self, ping_port_init_mock, ping_port_start_mock, data_clean_init_mock,data_clean_start_mock,
                 parse_args_mock, join_mock, start_mock, Controller_init_mock, try_to_connect_mock,
                 update_log_level_mock, daemonize_mock, perform_prestart_checks_mock,
-                resolve_ambari_config_mock, stop_mock, bind_signal_handlers_mock,
+                resolve_ambari_config_mock, stop_mock, bind_signal_handlers_mock, 
                 setup_logging_mock, socket_mock):
     data_clean_init_mock.return_value = None
     Controller_init_mock.return_value = None

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestNetUtil.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestNetUtil.py b/ambari-agent/src/test/python/ambari_agent/TestNetUtil.py
index 474548f..10ddca3 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestNetUtil.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestNetUtil.py
@@ -38,16 +38,16 @@ class TestNetUtil(unittest.TestCase):
 
     # test 200
     netutil = NetUtil.NetUtil()
-    self.assertTrue(netutil.checkURL("url")[0])
+    self.assertTrue(netutil.checkURL("url"))
 
     # test fail
     response.status = 404
-    self.assertFalse(netutil.checkURL("url")[0])
+    self.assertFalse(netutil.checkURL("url"))
 
     # test Exception
     response.status = 200
     httpsConMock.side_effect = Exception("test")
-    self.assertFalse(netutil.checkURL("url")[0])
+    self.assertFalse(netutil.checkURL("url"))
 
 
   @patch("time.sleep")
@@ -55,15 +55,15 @@ class TestNetUtil(unittest.TestCase):
 
     netutil = NetUtil.NetUtil()
     checkURL = MagicMock(name="checkURL")
-    checkURL.return_value = True, "test"
+    checkURL.return_value = True
     netutil.checkURL = checkURL
+    l = MagicMock()
 
     # one successful get
     self.assertEqual(0, netutil.try_to_connect("url", 10))
 
     # got successful after N retries
-    gets = [[True, ""], [False, ""], [False, ""]]
-
+    gets = [True, False, False]
     def side_effect(*args):
       return gets.pop()
     checkURL.side_effect = side_effect
@@ -71,5 +71,7 @@ class TestNetUtil(unittest.TestCase):
 
     # max retries
     checkURL.side_effect = None
-    checkURL.return_value = False, "test"
+    checkURL.return_value = False
     self.assertEqual(5, netutil.try_to_connect("url", 5))
+
+

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestPythonExecutor.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestPythonExecutor.py b/ambari-agent/src/test/python/ambari_agent/TestPythonExecutor.py
index d1bec5d..9e52eae 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestPythonExecutor.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestPythonExecutor.py
@@ -55,15 +55,12 @@ class TestPythonExecutor(TestCase):
     runShellKillPgrp_method.side_effect = lambda python : python.terminate()
     executor.runShellKillPgrp = runShellKillPgrp_method
     subproc_mock.returncode = None
-    callback_method = MagicMock()
     thread = Thread(target =  executor.run_file, args = ("fake_puppetFile",
-      ["arg1", "arg2"], "/fake_tmp_dir", tmpoutfile, tmperrfile,
-      PYTHON_TIMEOUT_SECONDS, tmpstrucout, "INFO", callback_method, '1'))
+      ["arg1", "arg2"], tmpoutfile, tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstrucout,"INFO"))
     thread.start()
     time.sleep(0.1)
     subproc_mock.finished_event.wait()
     self.assertEquals(subproc_mock.was_terminated, True, "Subprocess should be terminated due to timeout")
-    self.assertTrue(callback_method.called)
 
 
   def test_watchdog_2(self):
@@ -86,18 +83,16 @@ class TestPythonExecutor(TestCase):
     runShellKillPgrp_method.side_effect = lambda python : python.terminate()
     executor.runShellKillPgrp = runShellKillPgrp_method
     subproc_mock.returncode = 0
-    callback_method = MagicMock()
     thread = Thread(target =  executor.run_file, args = ("fake_puppetFile", ["arg1", "arg2"],
-                                                      "/fake_tmp_dir", tmpoutfile, tmperrfile,
-                                                      PYTHON_TIMEOUT_SECONDS, tmpstrucout,
-                                                      "INFO", callback_method, "1-1"))
+                                                      tmpoutfile, tmperrfile,
+                                                      PYTHON_TIMEOUT_SECONDS, tmpstrucout, "INFO"))
     thread.start()
     time.sleep(0.1)
     subproc_mock.should_finish_event.set()
     subproc_mock.finished_event.wait()
     self.assertEquals(subproc_mock.was_terminated, False, "Subprocess should not be terminated before timeout")
     self.assertEquals(subproc_mock.returncode, 0, "Subprocess should not be terminated before timeout")
-    self.assertTrue(callback_method.called)
+
 
   def test_execution_results(self):
     subproc_mock = self.Subprocess_mockup()
@@ -117,13 +112,10 @@ class TestPythonExecutor(TestCase):
     executor.runShellKillPgrp = runShellKillPgrp_method
     subproc_mock.returncode = 0
     subproc_mock.should_finish_event.set()
-    callback_method = MagicMock()
-    result = executor.run_file("file", ["arg1", "arg2"], "/fake_tmp_dir",
-                               tmpoutfile, tmperrfile, PYTHON_TIMEOUT_SECONDS,
-                               tmpstroutfile, "INFO", callback_method, "1-1")
+    result = executor.run_file("file", ["arg1", "arg2"], tmpoutfile, tmperrfile, PYTHON_TIMEOUT_SECONDS, tmpstroutfile, "INFO")
     self.assertEquals(result, {'exitcode': 0, 'stderr': 'Dummy err', 'stdout': 'Dummy output',
                                'structuredOut': {}})
-    self.assertTrue(callback_method.called)
+
 
   def test_is_successfull(self):
     executor = PythonExecutor("/tmp", AmbariConfig().getConfig())

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestRegistration.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestRegistration.py b/ambari-agent/src/test/python/ambari_agent/TestRegistration.py
index 2a9f716..b70ebbe 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestRegistration.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestRegistration.py
@@ -28,7 +28,8 @@ with patch("platform.linux_distribution", return_value = ('Suse','11','Final')):
   from ambari_agent.Register import Register
   from ambari_agent.AmbariConfig import AmbariConfig
   from ambari_agent.HostInfo import HostInfo
-  from ambari_commons import OSCheck, Firewall, FirewallChecks
+  from ambari_agent.HostInfo import FirewallChecks
+  from ambari_commons import OSCheck
 
 class TestRegistration(TestCase):
   @patch.object(FirewallChecks, "run_os_command")
@@ -59,7 +60,6 @@ class TestRegistration(TestCase):
     print data['agentEnv']['umask']
     self.assertEquals(not data['agentEnv']['umask']== "", True, "agents umask should not be empty")
     self.assertEquals(data['currentPingPort'] == 33777, True, "current ping port should be 33777")
-    self.assertEquals(data['prefix'], config.get('agent', 'prefix'), 'The prefix path does not match')
-    self.assertEquals(len(data), 9)
+    self.assertEquals(len(data), 8)
 
     os.remove(ver_file)

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
index 37a61c8..8b11063 100644
--- a/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
+++ b/ambari-agent/src/test/python/ambari_agent/TestSecurity.py
@@ -45,7 +45,7 @@ class TestSecurity(unittest.TestCase):
     out = StringIO.StringIO()
     sys.stdout = out
     # Create config
-    self.config = AmbariConfig()
+    self.config = AmbariConfig().getConfig()
     # Instantiate CachedHTTPSConnection (skip connect() call)
     with patch.object(security.VerifiedHTTPSConnection, "connect"):
       self.cachedHTTPSConnection = security.CachedHTTPSConnection(self.config)

http://git-wip-us.apache.org/repos/asf/ambari/blob/7e28d1e3/ambari-agent/src/test/python/resource_management/TestContentSources.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestContentSources.py b/ambari-agent/src/test/python/resource_management/TestContentSources.py
index 1c5e8a8..2527f30 100644
--- a/ambari-agent/src/test/python/resource_management/TestContentSources.py
+++ b/ambari-agent/src/test/python/resource_management/TestContentSources.py
@@ -27,7 +27,7 @@ from resource_management.core.source import DownloadSource
 from resource_management.core.source import Template
 from resource_management.core.source import InlineTemplate
 
-from ambari_jinja2 import UndefinedError, TemplateNotFound
+from jinja2 import UndefinedError, TemplateNotFound
 import urllib2
 import os
 


Mime
View raw message