ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jlun...@apache.org
Subject ambari git commit: AMBARI-12192: RU: fails with spnego-enabled cluster and HDFS HA (jluniya)
Date Sun, 28 Jun 2015 23:48:25 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 50cb277a5 -> e84f8891c


AMBARI-12192: RU: fails with spnego-enabled cluster and HDFS HA (jluniya)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/e84f8891
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/e84f8891
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/e84f8891

Branch: refs/heads/branch-2.1
Commit: e84f8891c37addc3a4ee6ff5c2faac60acc8664a
Parents: 50cb277
Author: Jayush Luniya <jluniya@hortonworks.com>
Authored: Sun Jun 28 16:47:21 2015 -0700
Committer: Jayush Luniya <jluniya@hortonworks.com>
Committed: Sun Jun 28 16:48:21 2015 -0700

----------------------------------------------------------------------
 .../resource_management/TestPackageResource.py  |  2 +-
 .../HDFS/2.1.0.2.0/package/files/checkWebUI.py  |  4 +-
 .../package/scripts/journalnode_upgrade.py      |  4 +-
 .../2.1.0.2.0/package/scripts/service_check.py  | 46 +++++++++++++-------
 .../HDFS/2.1.0.2.0/package/scripts/utils.py     | 29 +++++++-----
 .../stacks/2.0.6/HDFS/test_journalnode.py       | 15 +++----
 .../stacks/2.2/configs/journalnode-upgrade.json |  2 +-
 7 files changed, 60 insertions(+), 42 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/e84f8891/ambari-agent/src/test/python/resource_management/TestPackageResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestPackageResource.py b/ambari-agent/src/test/python/resource_management/TestPackageResource.py
index c67745d..18b2d00 100644
--- a/ambari-agent/src/test/python/resource_management/TestPackageResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestPackageResource.py
@@ -176,7 +176,7 @@ class TestPackageResource(TestCase):
               )
     self.assertEquals(shell_mock.call_args[0][0],
                       ['/usr/bin/yum', '-d', '0', '-e', '0', '-y', 'install',
-                       '--disablerepo=*',
+                       '--disablerepo=',
                        '--enablerepo=HDP-UTILS-2.2.0.1-885,HDP-2.2.0.1-885', 'some_package'])
 
   @patch.object(shell, "call", new = MagicMock(return_value=(0, None)))

http://git-wip-us.apache.org/repos/asf/ambari/blob/e84f8891/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files/checkWebUI.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files/checkWebUI.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files/checkWebUI.py
index 8ae6d2e..aa60ffc 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files/checkWebUI.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/files/checkWebUI.py
@@ -47,9 +47,7 @@ def main():
       httpCode = 404
 
     if httpCode != 200:
-      if not https:
-        print "Cannot access WEB UI on: http://" + host + ":" + port if not https.lower()
== "true" else "Cannot access WEB UI on: https://" + host + ":" + port
-
+      print "Cannot access WEB UI on: http://" + host + ":" + port if not https.lower() ==
"true" else "Cannot access WEB UI on: https://" + host + ":" + port
       exit(1)
 
 if __name__ == "__main__":

http://git-wip-us.apache.org/repos/asf/ambari/blob/e84f8891/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py
index a37412c..5e54593 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/journalnode_upgrade.py
@@ -56,7 +56,7 @@ def post_upgrade_check():
   nn_address = namenode_ha.get_address(NAMENODE_STATE.ACTIVE)
 
   nn_data = get_jmx_data(nn_address, 'org.apache.hadoop.hdfs.server.namenode.FSNamesystem',
'JournalTransactionInfo',
-                         namenode_ha.is_encrypted())
+                         namenode_ha.is_encrypted(), params.security_enabled)
   if not nn_data:
     raise Fail("Could not retrieve JournalTransactionInfo from JMX")
 
@@ -121,7 +121,7 @@ def ensure_jns_have_new_txn(nodes, last_txn_id):
         continue
 
       url = '%s://%s:%s' % (protocol, node, params.journalnode_port)
-      data = get_jmx_data(url, 'Journal-', 'LastWrittenTxId')
+      data = get_jmx_data(url, 'Journal-', 'LastWrittenTxId', params.https_only, params.security_enabled)
       if data:
         actual_txn_ids[node] = int(data)
         if actual_txn_ids[node] >= last_txn_id:

http://git-wip-us.apache.org/repos/asf/ambari/blob/e84f8891/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/service_check.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/service_check.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/service_check.py
index d66d70c..6ec3996 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/service_check.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/service_check.py
@@ -21,7 +21,8 @@ from resource_management import *
 from resource_management.core.shell import as_user
 from ambari_commons.os_family_impl import OsFamilyImpl
 from ambari_commons import OSConst
-
+from resource_management.libraries.functions.curl_krb_request import curl_krb_request
+from resource_management.core.logger import Logger
 
 class HdfsServiceCheck(Script):
   pass
@@ -68,21 +69,34 @@ class HdfsServiceCheckDefault(HdfsServiceCheck):
     params.HdfsResource(None, action="execute")
 
     if params.has_journalnode_hosts:
-      journalnode_port = params.journalnode_port
-      checkWebUIFileName = "checkWebUI.py"
-      checkWebUIFilePath = format("{tmp_dir}/{checkWebUIFileName}")
-      comma_sep_jn_hosts = ",".join(params.journalnode_hosts)
-      checkWebUICmd = format("python {checkWebUIFilePath} -m {comma_sep_jn_hosts} -p {journalnode_port}
-s {https_only}")
-      File(checkWebUIFilePath,
-           content=StaticFile(checkWebUIFileName),
-           mode=0775)
-
-      Execute(checkWebUICmd,
-              logoutput=True,
-              try_sleep=3,
-              tries=5,
-              user=params.smoke_user
-      )
+      if params.security_enabled:
+        for host in params.journalnode_hosts:
+          if params.https_only:
+            uri = format("https://{host}:{journalnode_port}")
+          else:
+            uri = format("http://{host}:{journalnode_port}")
+          response, errmsg, time_millis = curl_krb_request(params.tmp_dir, params.smoke_user_keytab,
+                                                           params.smokeuser_principal, uri,
"jn_service_check",
+                                                           params.kinit_path_local, False,
None, params.smoke_user)
+          if not response:
+            Logger.error("Cannot access WEB UI on: {0}. Error : {1}", uri, errmsg)
+            return 1
+      else:
+        journalnode_port = params.journalnode_port
+        checkWebUIFileName = "checkWebUI.py"
+        checkWebUIFilePath = format("{tmp_dir}/{checkWebUIFileName}")
+        comma_sep_jn_hosts = ",".join(params.journalnode_hosts)
+        checkWebUICmd = format("python {checkWebUIFilePath} -m {comma_sep_jn_hosts} -p {journalnode_port}
-s {https_only}")
+        File(checkWebUIFilePath,
+             content=StaticFile(checkWebUIFileName),
+             mode=0775)
+
+        Execute(checkWebUICmd,
+                logoutput=True,
+                try_sleep=3,
+                tries=5,
+                user=params.smoke_user
+        )
 
     if params.is_namenode_master:
       if params.has_zkfc_hosts:

http://git-wip-us.apache.org/repos/asf/ambari/blob/e84f8891/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py
b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py
index 93c8c8c..263bdec 100644
--- a/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py
+++ b/ambari-server/src/main/resources/common-services/HDFS/2.1.0.2.0/package/scripts/utils.py
@@ -29,6 +29,7 @@ from resource_management.core import shell
 from resource_management.core.shell import as_user, as_sudo
 from resource_management.core.exceptions import ComponentIsNotRunning
 from resource_management.core.logger import Logger
+from resource_management.libraries.functions.curl_krb_request import curl_krb_request
 
 from zkfc_slave import ZkfcSlave
 
@@ -261,7 +262,7 @@ def service(action=None, name=None, user=None, options="", create_pid_dir=False,
          action="delete",
     )
 
-def get_jmx_data(nn_address, modeler_type, metric, encrypted=False):
+def get_jmx_data(nn_address, modeler_type, metric, encrypted=False, security_enabled=False):
   """
   :param nn_address: Namenode Address, e.g., host:port, ** MAY ** be preceded with "http://"
or "https://" already.
   If not preceded, will use the encrypted param to determine.
@@ -281,17 +282,23 @@ def get_jmx_data(nn_address, modeler_type, metric, encrypted=False):
   nn_address = nn_address + "jmx"
   Logger.info("Retrieve modeler: %s, metric: %s from JMX endpoint %s" % (modeler_type, metric,
nn_address))
 
-  data = urllib2.urlopen(nn_address).read()
-  data_dict = json.loads(data)
+  if security_enabled:
+    import params
+    data, error_msg, time_millis = curl_krb_request(params.tmp_dir, params.smoke_user_keytab,
params.smokeuser_principal, nn_address,
+                            "jn_upgrade", params.kinit_path_local, False, None, params.smoke_user)
+  else:
+    data = urllib2.urlopen(nn_address).read()
   my_data = None
-  if data_dict:
-    for el in data_dict['beans']:
-      if el is not None and el['modelerType'] is not None and el['modelerType'].startswith(modeler_type):
-        if metric in el:
-          my_data = el[metric]
-          if my_data:
-            my_data = json.loads(str(my_data))
-            break
+  if data:
+    data_dict = json.loads(data)
+    if data_dict:
+      for el in data_dict['beans']:
+        if el is not None and el['modelerType'] is not None and el['modelerType'].startswith(modeler_type):
+          if metric in el:
+            my_data = el[metric]
+            if my_data:
+              my_data = json.loads(str(my_data))
+              break
   return my_data
 
 def get_port(address):

http://git-wip-us.apache.org/repos/asf/ambari/blob/e84f8891/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py
index 2e45f59..b7e99c1 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/HDFS/test_journalnode.py
@@ -261,7 +261,8 @@ class TestJournalnode(RMFTestCase):
 
   @patch('time.sleep')
   @patch("urllib2.urlopen")
-  def test_post_rolling_restart(self, urlopen_mock, time_mock):
+  @patch("utils.curl_krb_request")
+  def test_post_rolling_restart(self, curl_krb_request_mock, urlopen_mock, time_mock):
     # load the NN and JN JMX files so that the urllib2.urlopen mock has data
     # to return
     num_journalnodes = 3
@@ -300,9 +301,7 @@ class TestJournalnode(RMFTestCase):
     urlopen_mock.assert_called_with("http://c6407.ambari.apache.org:8480/jmx")
 
     url_stream_mock.reset_mock()
-    url_stream_mock.read.side_effect = (num_journalnodes * [namenode_jmx, journalnode_jmx])
-
-    urlopen_mock.return_value = url_stream_mock
+    curl_krb_request_mock.side_effect = (num_journalnodes * [(namenode_jmx, "", 1), (journalnode_jmx,
"", 1)])
 
     # now try with HDFS on SSL
     self.executeScript(self.COMMON_SERVICES_PACKAGE_DIR + "/scripts/journalnode.py",
@@ -313,10 +312,10 @@ class TestJournalnode(RMFTestCase):
       target = RMFTestCase.TARGET_COMMON_SERVICES )
 
     # ensure that the mock was called with the http-style version of the URL
-    urlopen_mock.assert_called
-    urlopen_mock.assert_called_with("https://c6407.ambari.apache.org:8481/jmx")
-
-
+    curl_krb_request_mock.assert_called
+    curl_krb_request_mock.assert_called_with("/tmp", "/etc/security/keytabs/smokeuser.headless.keytab",
+                                             "ambari-qa@EXAMPLE.COM", "https://c6407.ambari.apache.org:8481/jmx",
+                                             "jn_upgrade", "/usr/bin/kinit", False, None,
"ambari-qa")
 
   @patch('time.sleep')
   @patch("urllib2.urlopen")

http://git-wip-us.apache.org/repos/asf/ambari/blob/e84f8891/ambari-server/src/test/python/stacks/2.2/configs/journalnode-upgrade.json
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.2/configs/journalnode-upgrade.json b/ambari-server/src/test/python/stacks/2.2/configs/journalnode-upgrade.json
index 0419223..8b10691 100644
--- a/ambari-server/src/test/python/stacks/2.2/configs/journalnode-upgrade.json
+++ b/ambari-server/src/test/python/stacks/2.2/configs/journalnode-upgrade.json
@@ -1027,7 +1027,7 @@
             "content": "\n#\n#\n# Licensed to the Apache Software Foundation (ASF) under
one\n# or more contributor license agreements.  See the NOTICE file\n# distributed with this
work for additional information\n# regarding copyright ownership.  The ASF licenses this file\n#
to you under the Apache License, Version 2.0 (the\n# \"License\"); you may not use this file
except in compliance\n# with the License.  You may obtain a copy of the License at\n#\n# 
 http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed
to in writing,\n# software distributed under the License is distributed on an\n# \"AS IS\"
BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY\n# KIND, either express or implied.  See the
License for the\n# specific language governing permissions and limitations\n# under the License.\n#\n#\n#\n\n#
***** Set root logger level to DEBUG and its only appender to A.\nlog4j.logger.org.apache.pig=info,
A\n\n# ***** A is set to be a ConsoleAppender.\
 nlog4j.appender.A=org.apache.log4j.ConsoleAppender\n# ***** A uses PatternLayout.\nlog4j.appender.A.layout=org.apache.log4j.PatternLayout\nlog4j.appender.A.layout.ConversionPattern=%-4r
[%t] %-5p %c %x - %m%n"
         }, 
         "cluster-env": {
-            "security_enabled": "true",
+            "security_enabled": "false",
             "ignore_groupsusers_create": "false",
             "kerberos_domain": "EXAMPLE.COM",
             "user_group": "hadoop",


Mime
View raw message