ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From bhuvnesh2...@apache.org
Subject ambari git commit: AMBARI-15595: HAWQ - Add option to exchange keys during start or not (bhuvnesh2703)
Date Tue, 29 Mar 2016 13:59:30 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk abca3050c -> 6d668d295


AMBARI-15595: HAWQ - Add option to exchange keys during start or not (bhuvnesh2703)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/6d668d29
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/6d668d29
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/6d668d29

Branch: refs/heads/trunk
Commit: 6d668d29525e722b9408bef29594125e0ebd8793
Parents: abca305
Author: Bhuvnesh Chaudhary <bchaudhary@pivotal.io>
Authored: Tue Mar 29 07:00:31 2016 -0700
Committer: Bhuvnesh Chaudhary <bchaudhary@pivotal.io>
Committed: Tue Mar 29 07:00:31 2016 -0700

----------------------------------------------------------------------
 .../HAWQ/2.0.0/configuration/hawq-env.xml       | 16 +++++++-
 .../HAWQ/2.0.0/package/scripts/common.py        | 20 ++++++++++
 .../HAWQ/2.0.0/package/scripts/hawqmaster.py    | 10 ++++-
 .../HAWQ/2.0.0/package/scripts/master_helper.py | 41 +++++++-------------
 .../python/stacks/2.3/HAWQ/test_hawqmaster.py   | 18 +--------
 .../python/stacks/2.3/HAWQ/test_hawqstandby.py  |  7 ----
 .../python/stacks/2.3/configs/hawq_default.json |  3 +-
 ambari-web/app/data/HDP2.3/site_properties.js   |  9 ++++-
 8 files changed, 69 insertions(+), 55 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-env.xml
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-env.xml
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-env.xml
index fc5385f..d2da5b2 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-env.xml
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/configuration/hawq-env.xml
@@ -22,7 +22,7 @@
     <display-name>HAWQ System User Password</display-name>
     <value></value>
     <description>
-      Sets the HAWQ system user password. This password is used for ssh key exchange when
initializing a cluster or provisioning a new host.
+      Sets the HAWQ system user (gpadmin) password. This password is used for ssh key exchange
when initializing a cluster or provisioning a new host.
       Set when the HAWQ system user is created.
       When changing passwords after initial creation, first update the password directly
on the hosts, then enter a new hawq_password value.
     </description>
@@ -31,4 +31,16 @@
     </value-attributes>
   </property>
 
-</configuration>
\ No newline at end of file
+  <property>
+    <name>hawq_ssh_exkeys</name>
+    <display-name>Exchange SSH Keys</display-name>
+    <value>true</value>
+    <description>
+       Enable/Disable exchanging ssh keys for HAWQ system user (gpadmin) across all HAWQ
nodes during start of HAWQ Master. HAWQ system user
+       must be able to login to HAWQ nodes using passwordless ssh, else HAWQ operations will
fail. SSH keys must be exchanged during installation of
+       HAWQ service or when new HAWQ hosts are added. By default, ssh keys are exchanged
every time with HAWQ Master start/restart.
+       For larger clusters, this may be set to false temporarily to allow faster start time.
Valid values are true/false.
+    </description>
+  </property>
+
+</configuration>

http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
index d634fb2..c21dc88 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/common.py
@@ -21,6 +21,7 @@ import time
 import crypt
 import filecmp
 from resource_management.core.resources.system import Execute, Directory, File
+from resource_management.libraries.functions.default import default
 from resource_management.core.logger import Logger
 from resource_management.core.system import System
 from resource_management.core.exceptions import Fail
@@ -29,6 +30,7 @@ import xml.etree.ElementTree as ET
 
 import utils
 import hawq_constants
+import custom_params
 
 
 def setup_user():
@@ -279,6 +281,7 @@ def start_component(component_name, port, data_dir):
   """
   If data directory exists start the component, else initialize the component
   """
+  __check_dfs_truncate_enforced()
   if os.path.exists(os.path.join(data_dir, hawq_constants.postmaster_opts_filename)):
     return utils.exec_hawq_operation(hawq_constants.START,
                                      "{0} -a -v".format(component_name),
@@ -292,3 +295,20 @@ def stop_component(component_name, port, mode):
   utils.exec_hawq_operation(hawq_constants.STOP,
                             "{0} -M {1} -a -v".format(component_name, mode),
                             only_if=utils.chk_hawq_process_status_cmd(port, component_name))
+
+def __check_dfs_truncate_enforced():
+  """
+  If enforce_hdfs_truncate is set to True:
+    throw an ERROR, HAWQ components start should fail
+  Else:
+    throw a WARNING,
+  """
+  DFS_ALLOW_TRUNCATE_WARNING_MSG = "It is recommended to set dfs.allow.truncate as true in
hdfs-site.xml configuration file, currently it is set to false. Please review HAWQ installation
guide for more information."
+
+  # Check if dfs.allow.truncate exists in hdfs-site.xml and throw appropriate exception if
not set to True
+  dfs_allow_truncate = default("/configurations/hdfs-site/dfs.allow.truncate", None)
+  if dfs_allow_truncate is None or str(dfs_allow_truncate).lower() != 'true':
+    if custom_params.enforce_hdfs_truncate:
+      raise Fail("**ERROR**: {0}".format(DFS_ALLOW_TRUNCATE_WARNING_MSG))
+    else:
+      Logger.error("**WARNING**: {0}".format(DFS_ALLOW_TRUNCATE_WARNING_MSG))

http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
index 7c3a832..4edf64f 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/hawqmaster.py
@@ -20,6 +20,8 @@ from resource_management import Script
 from resource_management.core.resources.system import Execute
 from resource_management.core.logger import Logger
 from resource_management.libraries.functions.check_process_status import check_process_status
+from resource_management.libraries.functions.default import default
+from resource_management.core.source import InlineTemplate
 try:
     from resource_management.libraries.functions import stack_select as hadoop_select
 except ImportError:
@@ -50,7 +52,11 @@ class HawqMaster(Script):
     import params
     self.configure(env)
     common.validate_configuration()
-    master_helper.setup_passwordless_ssh()
+    exchange_ssh_keys = default('/configurations/hawq-env/hawq_ssh_exkeys', None)
+    if exchange_ssh_keys is None or str(exchange_ssh_keys).lower() == 'false':
+      Logger.info("Skipping ssh key exchange with HAWQ hosts as hawq_ssh_exkeys is either
set to false or is not available in hawq-env.xml")
+    else:
+      master_helper.setup_passwordless_ssh()
     common.start_component(hawq_constants.MASTER, params.hawq_master_address_port, params.hawq_master_dir)
 
   def stop(self, env):
@@ -73,7 +79,9 @@ class HawqMaster(Script):
     exec_psql_cmd(cmd, params.hawqmaster_host, params.hawq_master_address_port)
 
   def run_hawq_check(self, env):
+    import params
     Logger.info("Executing HAWQ Check ...")
+    params.File(hawq_constants.hawq_hosts_file, content=InlineTemplate("{% for host in hawq_all_hosts
%}{{host}}\n{% endfor %}"))
     Execute("source {0} && hawq check -f {1} --hadoop {2} --config {3}".format(hawq_constants.hawq_greenplum_path_file,
hawq_constants.hawq_hosts_file, hadoop_select.get_hadoop_dir('home'), hawq_constants.hawq_check_file),
             user=hawq_constants.hawq_user,
             timeout=hawq_constants.default_exec_timeout)

http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
index 9d8b32d..87d3c8a 100644
--- a/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
+++ b/ambari-server/src/main/resources/common-services/HAWQ/2.0.0/package/scripts/master_helper.py
@@ -22,6 +22,7 @@ from resource_management.core.source import InlineTemplate
 from resource_management.core.logger import Logger
 from resource_management.libraries.functions.format import format
 from resource_management.libraries.functions.default import default
+from resource_management.core.exceptions import Fail
 
 import utils
 import common
@@ -39,17 +40,27 @@ def __setup_master_specific_conf_files():
   params.File(hawq_constants.hawq_slaves_file,
               content=InlineTemplate("{% for host in hawqsegment_hosts %}{{host}}\n{% endfor
%}"))
 
-  params.File(hawq_constants.hawq_hosts_file,
-              content=InlineTemplate("{% for host in hawq_all_hosts %}{{host}}\n{% endfor
%}"))
-
 
 def setup_passwordless_ssh():
   """
   Exchanges ssh keys to setup passwordless ssh for the hawq_user between the HAWQ Master
and the HAWQ Segment nodes
   """
   import params
-  utils.exec_hawq_operation("ssh-exkeys", format('-f {hawq_hosts_file} -p {hawq_password!p}',
hawq_hosts_file=hawq_constants.hawq_hosts_file, hawq_password=params.hawq_password))
 
+  failed_hosts = []
+  for host in params.hawq_all_hosts:
+    try:
+      utils.exec_hawq_operation("ssh-exkeys", format('-h {hawq_host} -p {hawq_password!p}',
hawq_host=host, hawq_password=params.hawq_password))
+    except:
+      failed_hosts.append(host)
+
+  failed_hosts_cnt = len(failed_hosts)
+  if failed_hosts_cnt > 0:
+    DEBUG_HELP_MSG = "Please verify the logs below to debug the cause of failure."
+    if failed_hosts_cnt == len(params.hawq_all_hosts):
+      raise Fail("Setting up passwordless ssh failed for all the HAWQ hosts. {0}".format(DEBUG_HELP_MSG))
+    else:
+      Logger.error("**WARNING**: Setting up passwordless ssh failed with the hosts below,
proceeding with HAWQ Master start:\n{0}\n\n{1}".format("\n".join(failed_hosts), DEBUG_HELP_MSG))
 
 def configure_master():
   """
@@ -61,7 +72,6 @@ def configure_master():
   __setup_master_specific_conf_files()
   common.create_master_dir(params.hawq_master_dir)
   common.create_temp_dirs(params.hawq_master_temp_dirs)
-  __check_dfs_truncate_enforced()
 
 def __setup_hdfs_dirs():
   """
@@ -80,24 +90,3 @@ def __setup_hdfs_dirs():
                         recursive_chown = True,
                         mode=0755)
   params.HdfsResource(None, action="execute")
-
-def __check_dfs_truncate_enforced():
-  """
-  If enforce_hdfs_truncate is set to True:
-    throw an ERROR, HAWQMASTER or HAWQSTANDBY start should fail
-  Else:
-    throw a WARNING,
-  """
-  import custom_params
-
-  DFS_ALLOW_TRUNCATE_EXCEPTION_MESSAGE = "dfs.allow.truncate property in hdfs-site.xml configuration
file should be set to True. Please review HAWQ installation guide for more information."
-
-  # Check if dfs.allow.truncate exists in hdfs-site.xml and throw appropriate exception if
not set to True
-  dfs_allow_truncate = default('/configurations/hdfs-site/dfs.allow.truncate', None)
-
-  if dfs_allow_truncate is None or str(dfs_allow_truncate).lower() != 'true':
-    if custom_params.enforce_hdfs_truncate:
-      Logger.error("**ERROR**: {0}".format(DFS_ALLOW_TRUNCATE_EXCEPTION_MESSAGE))
-      sys.exit(1)
-    else:
-      Logger.warning("**WARNING**: {0}".format(DFS_ALLOW_TRUNCATE_EXCEPTION_MESSAGE))

http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py
index 4dc3cc8..20624ec 100644
--- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py
+++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqmaster.py
@@ -87,13 +87,6 @@ class TestHawqMaster(RMFTestCase):
         mode = 0644
         )
 
-    self.assertResourceCalled('File', '/usr/local/hawq/etc/hawq_hosts',
-        content = InlineTemplate('c6401.ambari.apache.org\nc6402.ambari.apache.org\nc6403.ambari.apache.org\n\n'),
-        group = self.GPADMIN,
-        owner = self.GPADMIN,
-        mode = 0644
-        )
-
     self.assertResourceCalled('Directory', '/data/hawq/master',
         group = self.GPADMIN,
         owner = self.GPADMIN,
@@ -113,8 +106,7 @@ class TestHawqMaster(RMFTestCase):
 
 
   @patch ('hawqmaster.common.__set_osparams')
-  @patch ('hawqmaster.master_helper.__check_dfs_truncate_enforced')
-  def test_configure_default(self, set_dfs_truncate_mock, set_osparams_mock):
+  def test_configure_default(self, set_osparams_mock):
 
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + '/scripts/hawqmaster.py',
         classname = 'HawqMaster',
@@ -155,14 +147,6 @@ class TestHawqMaster(RMFTestCase):
 
     self.__asserts_for_configure()
 
-    self.assertResourceCalled('Execute', 'source /usr/local/hawq/greenplum_path.sh &&
hawq ssh-exkeys -f /usr/local/hawq/etc/hawq_hosts -p gpadmin',
-        logoutput = True,
-        not_if = None,
-        only_if = None,
-        user = self.GPADMIN,
-        timeout = 900
-        )
-
     self.assertResourceCalled('HdfsResource', '/hawq_default',
         immutable_paths = self.DEFAULT_IMMUTABLE_PATHS,
         default_fs = u'hdfs://c6401.ambari.apache.org:8020',

http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqstandby.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqstandby.py b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqstandby.py
index e98b253..be5989d 100644
--- a/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqstandby.py
+++ b/ambari-server/src/test/python/stacks/2.3/HAWQ/test_hawqstandby.py
@@ -85,13 +85,6 @@ class TestHawqStandby(RMFTestCase):
         mode = 0644
         )
 
-    self.assertResourceCalled('File', '/usr/local/hawq/etc/hawq_hosts',
-        content = InlineTemplate('c6401.ambari.apache.org\nc6402.ambari.apache.org\nc6403.ambari.apache.org\n\n'),
-        group = self.GPADMIN,
-        owner = self.GPADMIN,
-        mode = 0644
-        )
-
     self.assertResourceCalled('Directory', '/data/hawq/master',
         group = self.GPADMIN,
         owner = self.GPADMIN,

http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json b/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json
index 99d43cb..36513c5 100644
--- a/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json
+++ b/ambari-server/src/test/python/stacks/2.3/configs/hawq_default.json
@@ -439,7 +439,8 @@
         }, 
         "ranger-hbase-plugin-properties": {}, 
         "hawq-env": {
-            "hawq_password": "gpadmin"
+            "hawq_password": "gpadmin",
+            "hawq_ssh_exkeys": "false"
         }, 
         "core-site": {
             "ipc.server.listen.queue.size": "3300", 

http://git-wip-us.apache.org/repos/asf/ambari/blob/6d668d29/ambari-web/app/data/HDP2.3/site_properties.js
----------------------------------------------------------------------
diff --git a/ambari-web/app/data/HDP2.3/site_properties.js b/ambari-web/app/data/HDP2.3/site_properties.js
index e706d40..e04d060 100644
--- a/ambari-web/app/data/HDP2.3/site_properties.js
+++ b/ambari-web/app/data/HDP2.3/site_properties.js
@@ -340,13 +340,20 @@ hdp23properties.push({
     "index": 9
   },
   {
-    "name": "hawq_password",
+    "name": "hawq_ssh_exkeys",
     "filename": "hawq-env.xml",
     "category": "General",
     "serviceName": "HAWQ",
     "index": 10
   },
   {
+    "name": "hawq_password",
+    "filename": "hawq-env.xml",
+    "category": "General",
+    "serviceName": "HAWQ",
+    "index": 11
+  },
+  {
     "name": "content",
     "serviceName": "HAWQ",
     "filename": "hawq-check-env.xml",


Mime
View raw message