ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From rle...@apache.org
Subject [1/2] ambari git commit: Revert "AMBARI-10562. Summary:Issues when log/pid dirs are customized - Kafka and Knox (Emil Anca via rlevas)"
Date Fri, 24 Apr 2015 15:12:49 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk bdb46fec1 -> 239d3ac75


Revert "AMBARI-10562. Summary:Issues when log/pid dirs are customized - Kafka and Knox (Emil
Anca via rlevas)"

This reverts commit 52b5022bc58906a3579d291b078d5a49b240ce6e.


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/b971804e
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/b971804e
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/b971804e

Branch: refs/heads/trunk
Commit: b971804e5d0d479aa180d0921a42834e2b4f1efa
Parents: bdb46fe
Author: Robert Levas <rlevas@hortonworks.com>
Authored: Fri Apr 24 11:10:53 2015 -0400
Committer: Robert Levas <rlevas@hortonworks.com>
Committed: Fri Apr 24 11:10:53 2015 -0400

----------------------------------------------------------------------
 .../KAFKA/0.8.1.2.2/package/scripts/kafka.py    | 75 +--------------
 .../KAFKA/0.8.1.2.2/package/scripts/params.py   |  5 +-
 .../0.5.0.2.2/package/scripts/knox_gateway.py   |  5 -
 .../KNOX/0.5.0.2.2/package/scripts/params.py    |  2 -
 .../stacks/2.2/KAFKA/test_kafka_broker.py       | 43 +--------
 .../python/stacks/2.2/KNOX/test_knox_gateway.py | 96 --------------------
 6 files changed, 4 insertions(+), 222 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/b971804e/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
index 6668c4e..0170116 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/kafka.py
@@ -20,7 +20,7 @@ limitations under the License.
 
 from resource_management import *
 from properties_config import properties_config
-import sys, os
+import sys
 from copy import deepcopy
 
 def kafka():
@@ -71,82 +71,9 @@ def kafka():
              content=params.log4j_props
          )
 
-    setup_symlink(params.kafka_managed_pid_dir, params.kafka_pid_dir)
-    setup_symlink(params.kafka_managed_log_dir, params.kafka_log_dir)
-
 
 def mutable_config_dict(kafka_broker_config):
     kafka_server_config = {}
     for key, value in kafka_broker_config.iteritems():
         kafka_server_config[key] = value
     return kafka_server_config
-
-# Used to workaround the hardcoded pid/log dir used on the kafka bash process launcher
-def setup_symlink(kafka_managed_dir, kafka_ambari_managed_dir):
-  import params
-  backup_folder_path = None
-  backup_folder_suffix = "_tmp"
-  if kafka_ambari_managed_dir != kafka_managed_dir:
-    if os.path.exists(kafka_managed_dir) and not os.path.islink(kafka_managed_dir):
-
-      # Backup existing data before delete if config is changed repeatedly to/from default
location at any point in time time, as there may be relevant contents (historic logs)
-      backup_folder_path = backup_dir_contents(kafka_managed_dir, backup_folder_suffix)
-
-      Directory(kafka_managed_dir,
-                action="delete",
-                recursive=True)
-
-    elif os.path.islink(kafka_managed_dir) and os.path.realpath(kafka_managed_dir) != kafka_ambari_managed_dir:
-      Link(kafka_managed_dir,
-           action="delete")
-
-    if not os.path.islink(kafka_managed_dir):
-      Link(kafka_managed_dir,
-           to=kafka_ambari_managed_dir)
-
-  elif os.path.islink(kafka_managed_dir): # If config is changed and coincides with the kafka
managed dir, remove the symlink and physically create the folder
-    Link(kafka_managed_dir,
-         action="delete")
-
-    Directory(kafka_managed_dir,
-              mode=0755,
-              cd_access='a',
-              owner=params.kafka_user,
-              group=params.user_group,
-              recursive=True
-    )
-
-  if backup_folder_path:
-    # Restore backed up files to current relevant dirs if needed - will be triggered only
when changing to/from default path;
-    for file in os.listdir(backup_folder_path):
-      File(os.path.join(kafka_managed_dir,file),
-           owner=params.kafka_user,
-           content = StaticFile(os.path.join(backup_folder_path,file)))
-
-    # Clean up backed up folder
-    Directory(backup_folder_path,
-              action="delete",
-              recursive=True)
-
-
-# Uses agent temp dir to store backup files
-def backup_dir_contents(dir_path, backup_folder_suffix):
-  import params
-  backup_destination_path = params.tmp_dir + os.path.normpath(dir_path)+backup_folder_suffix
-  Directory(backup_destination_path,
-            mode=0755,
-            cd_access='a',
-            owner=params.kafka_user,
-            group=params.user_group,
-            recursive=True
-  )
-  # Safely copy top-level contents to backup folder
-  for file in os.listdir(dir_path):
-    File(os.path.join(backup_destination_path, file),
-         owner=params.kafka_user,
-         content = StaticFile(os.path.join(dir_path,file)))
-
-  return backup_destination_path
-
-
-

http://git-wip-us.apache.org/repos/asf/ambari/blob/b971804e/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
index e9f8f56..74c189e 100644
--- a/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/KAFKA/0.8.1.2.2/package/scripts/params.py
@@ -26,7 +26,7 @@ import status_params
 
 # server configurations
 config = Script.get_config()
-tmp_dir = Script.get_tmp_dir()
+
 stack_name = default("/hostLevelParams/stack_name", None)
 
 version = default("/commandParams/version", None)
@@ -48,9 +48,6 @@ kafka_user = config['configurations']['kafka-env']['kafka_user']
 kafka_log_dir = config['configurations']['kafka-env']['kafka_log_dir']
 kafka_pid_dir = status_params.kafka_pid_dir
 kafka_pid_file = kafka_pid_dir+"/kafka.pid"
-# This is hardcoded on the kafka bash process lifecycle on which we have no control over
-kafka_managed_pid_dir = "/var/run/kafka"
-kafka_managed_log_dir = "/var/log/kafka"
 hostname = config['hostname']
 user_group = config['configurations']['cluster-env']['user_group']
 java64_home = config['hostLevelParams']['java_home']

http://git-wip-us.apache.org/repos/asf/ambari/blob/b971804e/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
index 27a302f..9e20499 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/knox_gateway.py
@@ -81,11 +81,6 @@ class KnoxGateway(Script):
     daemon_cmd = format('{knox_bin} start')
     no_op_test = format('ls {knox_pid_file} >/dev/null 2>&1 && ps -p `cat
{knox_pid_file}` >/dev/null 2>&1')
     setup_ranger_knox()
-    # Used to setup symlink, needed to update the knox managed symlink, in case of custom
locations
-    if os.path.islink(params.knox_managed_pid_symlink) and os.path.realpath(params.knox_managed_pid_symlink)
!= params.knox_pid_dir:
-      os.unlink(params.knox_managed_pid_symlink)
-      os.symlink(params.knox_pid_dir, params.knox_managed_pid_symlink)
-
     Execute(daemon_cmd,
             user=params.knox_user,
             environment={'JAVA_HOME': params.java_home},

http://git-wip-us.apache.org/repos/asf/ambari/blob/b971804e/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py
b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py
index 344dae9..a04842e 100644
--- a/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py
+++ b/ambari-server/src/main/resources/common-services/KNOX/0.5.0.2.2/package/scripts/params.py
@@ -110,8 +110,6 @@ if has_oozie:
     if 'oozie.base.url' in config['configurations']['oozie-site']:
         oozie_server_port = get_port_from_url(config['configurations']['oozie-site']['oozie.base.url'])
 
-# Knox managed properties
-knox_managed_pid_symlink= "/usr/hdp/current/knox-server/pids"
 
 # server configurations
 knox_pid_dir = status_params.knox_pid_dir

http://git-wip-us.apache.org/repos/asf/ambari/blob/b971804e/ambari-server/src/test/python/stacks/2.2/KAFKA/test_kafka_broker.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/KAFKA/test_kafka_broker.py b/ambari-server/src/test/python/stacks/2.2/KAFKA/test_kafka_broker.py
index 151335c..2b6f7e4 100644
--- a/ambari-server/src/test/python/stacks/2.2/KAFKA/test_kafka_broker.py
+++ b/ambari-server/src/test/python/stacks/2.2/KAFKA/test_kafka_broker.py
@@ -19,7 +19,7 @@ limitations under the License.
 '''
 import json
 from stacks.utils.RMFTestCase import *
-from mock.mock import patch
+
 
 class TestKafkaBroker(RMFTestCase):
   COMMON_SERVICES_PACKAGE_DIR = "KAFKA/0.8.1.2.2/package"
@@ -58,45 +58,6 @@ class TestKafkaBroker(RMFTestCase):
                               cd_access = 'a'
     )
 
-  @patch("os.path.islink")
-  @patch("os.path.realpath")
-  def test_configure_custom_paths_default(self, realpath_mock, islink_mock):
-
-    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/kafka_broker.py",
-                       classname = "KafkaBroker",
-                       command = "configure",
-                       config_file="default_custom_path_config.json",
-                       hdp_stack_version = self.STACK_VERSION,
-                       target = RMFTestCase.TARGET_COMMON_SERVICES
-    )
-
-    self.assertResourceCalled('Directory', '/customdisk/var/log/kafka',
-                              owner = 'kafka',
-                              group = 'hadoop',
-                              recursive = True,
-                              mode = 0755,
-                              cd_access = 'a'
-    )
-
-    self.assertResourceCalled('Directory', '/customdisk/var/run/kafka',
-                              owner = 'kafka',
-                              group = 'hadoop',
-                              recursive = True,
-                              mode = 0755,
-                              cd_access = 'a'
-    )
-
-    self.assertResourceCalled('Directory', '/etc/kafka/conf',
-                              owner = 'kafka',
-                              group = 'hadoop',
-                              recursive = True,
-                              mode = 0755,
-                              cd_access = 'a'
-    )
-
-    self.assertTrue(islink_mock.called)
-    self.assertTrue(realpath_mock.called)
-
   def test_pre_rolling_restart(self):
     config_file = self.get_src_folder()+"/test/python/stacks/2.2/configs/default.json"
     with open(config_file, "r") as f:
@@ -111,4 +72,4 @@ class TestKafkaBroker(RMFTestCase):
                        target = RMFTestCase.TARGET_COMMON_SERVICES)
     self.assertResourceCalled('Execute',
                               'hdp-select set kafka-broker %s' % version,)
-    self.assertNoMoreResources()
\ No newline at end of file
+    self.assertNoMoreResources()

http://git-wip-us.apache.org/repos/asf/ambari/blob/b971804e/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
index 82bee2a..37bb66b 100644
--- a/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
+++ b/ambari-server/src/test/python/stacks/2.2/KNOX/test_knox_gateway.py
@@ -230,99 +230,3 @@ class TestKnoxGateway(RMFTestCase):
     self.assertTrue(tarfile_open_mock.called)
 
     self.assertResourceCalled("Execute", "hdp-select set knox-server %s" % version)
-
-  @patch("os.path.islink")
-  @patch("os.path.realpath")
-  @patch("os.unlink")
-  @patch("os.symlink")
-  def test_start_default(self, symlink_mock, unlink_mock, realpath_mock, islink_mock):
-
-
-    self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/knox_gateway.py",
-                       classname = "KnoxGateway",
-                       command = "start",
-                       config_file="default.json",
-                       hdp_stack_version = self.STACK_VERSION,
-                       target = RMFTestCase.TARGET_COMMON_SERVICES)
-
-
-    self.assertResourceCalled('Directory', '/var/lib/knox/data',
-                              owner = 'knox',
-                              group = 'knox',
-                              recursive = True
-    )
-    self.assertResourceCalled('Directory', '/var/log/knox',
-                              owner = 'knox',
-                              group = 'knox',
-                              recursive = True
-    )
-    self.assertResourceCalled('Directory', '/var/run/knox',
-                              owner = 'knox',
-                              group = 'knox',
-                              recursive = True
-    )
-    self.assertResourceCalled('Directory', '/etc/knox/conf',
-                              owner = 'knox',
-                              group = 'knox',
-                              recursive = True
-    )
-
-    self.assertResourceCalled('XmlConfig', 'gateway-site.xml',
-                              owner = 'knox',
-                              group = 'knox',
-                              conf_dir = '/etc/knox/conf',
-                              configurations = self.getConfig()['configurations']['gateway-site'],
-                              configuration_attributes = self.getConfig()['configuration_attributes']['gateway-site']
-    )
-
-    self.assertResourceCalled('File', '/etc/knox/conf/gateway-log4j.properties',
-                              mode=0644,
-                              group='knox',
-                              owner = 'knox',
-                              content = self.getConfig()['configurations']['gateway-log4j']['content']
-    )
-    self.assertResourceCalled('File', '/etc/knox/conf/topologies/default.xml',
-                              group='knox',
-                              owner = 'knox',
-                              content = InlineTemplate(self.getConfig()['configurations']['topology']['content'])
-    )
-    self.assertResourceCalled('Execute', ('chown',
-                                          '-R',
-                                          'knox:knox',
-                                          '/var/lib/knox/data',
-                                          '/var/log/knox',
-                                          '/var/run/knox',
-                                          '/etc/knox/conf'),
-                              sudo = True,
-                              )
-    self.assertResourceCalled('Execute', '/usr/hdp/current/knox-server/bin/knoxcli.sh create-master
--master sa',
-                              environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
-                              not_if = "ambari-sudo.sh su knox -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]test
-f /var/lib/knox/data/security/master'",
-                              user = 'knox',
-                              )
-    self.assertResourceCalled('Execute', '/usr/hdp/current/knox-server/bin/knoxcli.sh create-cert
--hostname c6401.ambari.apache.org',
-                              environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
-                              not_if = "ambari-sudo.sh su knox -l -s /bin/bash -c '[RMF_EXPORT_PLACEHOLDER]test
-f /var/lib/knox/data/security/keystores/gateway.jks'",
-                              user = 'knox',
-                              )
-    self.assertResourceCalled('File', '/etc/knox/conf/ldap-log4j.properties',
-                              mode=0644,
-                              group='knox',
-                              owner = 'knox',
-                              content = self.getConfig()['configurations']['ldap-log4j']['content']
-    )
-    self.assertResourceCalled('File', '/etc/knox/conf/users.ldif',
-                              mode=0644,
-                              group='knox',
-                              owner = 'knox',
-                              content = self.getConfig()['configurations']['users-ldif']['content']
-    )
-
-    self.assertResourceCalled("Execute", "/usr/hdp/current/knox-server/bin/gateway.sh start",
-                              environment = {'JAVA_HOME': u'/usr/jdk64/jdk1.7.0_45'},
-                              not_if = u'ls /var/run/knox/gateway.pid >/dev/null 2>&1
&& ps -p `cat /var/run/knox/gateway.pid` >/dev/null 2>&1',
-                              user = u'knox',)
-    self.assertTrue(islink_mock.called)
-    self.assertTrue(realpath_mock.called)
-    self.assertNoMoreResources()
-


Mime
View raw message