ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From d...@apache.org
Subject ambari git commit: AMBARI-12935 Ambari is overwriting ownership of yarn.local.dir with yarn:hadoop in Kerberized cluster (dsen)
Date Mon, 31 Aug 2015 12:17:07 GMT
Repository: ambari
Updated Branches:
  refs/heads/branch-2.1 bd0ff5a7e -> a79d5ae3e


AMBARI-12935 Ambari is overwriting ownership of yarn.local.dir with yarn:hadoop in Kerberized
cluster (dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/a79d5ae3
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/a79d5ae3
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/a79d5ae3

Branch: refs/heads/branch-2.1
Commit: a79d5ae3eb95ad9a3bde5b0784ed0ac251af1672
Parents: bd0ff5a
Author: Dmytro Sen <dsen@apache.org>
Authored: Mon Aug 31 15:13:48 2015 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Mon Aug 31 15:16:58 2015 +0300

----------------------------------------------------------------------
 .../2.1.0.2.0/package/scripts/params_linux.py   |  7 ++
 .../YARN/2.1.0.2.0/package/scripts/yarn.py      | 72 ++++++++++++++------
 .../stacks/2.0.6/YARN/test_nodemanager.py       | 22 +++---
 .../stacks/2.0.6/YARN/test_resourcemanager.py   |  6 ++
 4 files changed, 73 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/a79d5ae3/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
index 3d45d21..70ae4b8 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/params_linux.py
@@ -118,6 +118,12 @@ hdfs_user = config['configurations']['hadoop-env']['hdfs_user']
 smokeuser = config['configurations']['cluster-env']['smokeuser']
 smokeuser_principal = config['configurations']['cluster-env']['smokeuser_principal_name']
 security_enabled = config['configurations']['cluster-env']['security_enabled']
+nm_security_marker = '/var/lib/hadoop-yarn/nm_security_enabled'
+current_nm_security_state = os.path.isfile(nm_security_marker)
+toggle_nm_security = (current_nm_security_state and not security_enabled) or (not current_nm_security_state
and security_enabled)
+rm_security_marker = "/var/lib/hadoop-yarn/rm_security_enabled"
+current_rm_security_state = os.path.isfile(rm_security_marker)
+toggle_rm_security = (current_rm_security_state and not security_enabled) or (not current_rm_security_state
and security_enabled)
 smoke_user_keytab = config['configurations']['cluster-env']['smokeuser_keytab']
 yarn_executor_container_group = config['configurations']['yarn-site']['yarn.nodemanager.linux-container-executor.group']
 yarn_nodemanager_container_executor_class =  config['configurations']['yarn-site']['yarn.nodemanager.container-executor.class']
@@ -145,6 +151,7 @@ mapred_pid_dir_prefix = status_params.mapred_pid_dir_prefix
 mapred_log_dir_prefix = config['configurations']['mapred-env']['mapred_log_dir_prefix']
 mapred_env_sh_template = config['configurations']['mapred-env']['content']
 yarn_env_sh_template = config['configurations']['yarn-env']['content']
+yarn_nodemanager_recovery_dir = default('/configurations/yarn-site/yarn.nodemanager.recovery.dir',
None)
 
 if len(rm_hosts) > 1:
   additional_rm_host = rm_hosts[1]

http://git-wip-us.apache.org/repos/asf/ambari/blob/a79d5ae3/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index 30629be..bea6954 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -98,31 +98,47 @@ def yarn(name = None):
               )
 
   if name == "nodemanager":
-    Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','),
-              owner=params.yarn_user,
-              group=params.user_group,
-              recursive=True,
-              cd_access="a",
-              ignore_failures=True,
-              mode=0775
-              )
 
-    smokeuser_directories = [os.path.join(dir, 'usercache' ,params.smokeuser)
-                             for dir in params.nm_local_dirs.split(',')]
+    # First start after enabling/disabling security
+    if params.toggle_nm_security:
+      Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','),
+                action='delete'
+      )
 
-    if not params.security_enabled:
-      for directory in smokeuser_directories:
-        Execute(('chown', '-R', params.yarn_user, directory),
-            only_if=format("test -d {directory}"),
-            sudo=True)
+      # If yarn.nodemanager.recovery.dir exists, remove this dir
+      if params.yarn_nodemanager_recovery_dir:
+        Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
+                  action='delete'
+        )
 
+      # Setting NM marker file
+      if params.security_enabled:
+        File(params.nm_security_marker,
+             content="Marker file to track first start after enabling/disabling security.
"
+                     "During first start yarn local, log dirs are removed and recreated"
+             )
+      elif not params.security_enabled:
+        File(params.nm_security_marker, action="delete")
+
+
+    if not params.security_enabled or params.toggle_nm_security:
+      Directory(params.nm_local_dirs.split(',') + params.nm_log_dirs.split(','),
+                owner=params.yarn_user,
+                group=params.user_group,
+                recursive=True,
+                cd_access="a",
+                ignore_failures=True,
+                mode=0775
+                )
+
+  if params.yarn_nodemanager_recovery_dir:
+    Directory(InlineTemplate(params.yarn_nodemanager_recovery_dir).get_content(),
+              owner=params.yarn_user,
+              group=params.user_group,
+              recursive=True,
+              mode=0755
+    )
 
-    if params.security_enabled:
-      for directory in smokeuser_directories:
-        Execute(('chown', '-R', params.smokeuser, directory),
-                only_if=format("test -d {directory}"),
-                sudo=True,
-        )
   Directory([params.yarn_pid_dir_prefix, params.yarn_pid_dir, params.yarn_log_dir],
             owner=params.yarn_user,
             group=params.user_group,
@@ -206,6 +222,20 @@ def yarn(name = None):
                            mode=0700
       )
       params.HdfsResource(None, action="execute")
+    if params.toggle_rm_security:
+      Execute('yarn resourcemanager -format-state-store', user = params.yarn_user,
+      )
+      # Setting RM marker file
+      if params.security_enabled:
+        File(params.rm_security_marker,
+             content="Marker file to track first start after enabling/disabling security.
"
+                     "During first start ResourceManager state store is formatted"
+        )
+      elif not params.security_enabled:
+        File(params.rm_security_marker, action="delete")
+
+
+
   elif name == 'apptimelineserver':
     Directory(params.ats_leveldb_dir,
        owner=params.yarn_user,

http://git-wip-us.apache.org/repos/asf/ambari/blob/a79d5ae3/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
index 613ab69..6794097 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_nodemanager.py
@@ -167,15 +167,6 @@ class TestNodeManager(RMFTestCase):
                               mode = 0775,
                               cd_access='a'
                               )
-    self.assertResourceCalled('Execute', ('chown', '-R', u'yarn', u'/hadoop/yarn/local/usercache/ambari-qa'),
-                              sudo = True,
-                              only_if = 'test -d /hadoop/yarn/local/usercache/ambari-qa',
-                              )
-
-    self.assertResourceCalled('Execute', ('chown', '-R', u'yarn', u'/hadoop/yarn/local1/usercache/ambari-qa'),
-                              sudo = True,
-                              only_if = 'test -d /hadoop/yarn/local1/usercache/ambari-qa',
-                              )
     self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
       owner = 'yarn',
       group = 'hadoop',
@@ -334,6 +325,15 @@ class TestNodeManager(RMFTestCase):
 
   def assert_configure_secured(self):
     self.assertResourceCalled('Directory', '/hadoop/yarn/local',
+                              action = ['delete']
+    )
+    self.assertResourceCalled('Directory', '/hadoop/yarn/log',
+                              action = ['delete']
+    )
+    self.assertResourceCalled('File', '/var/lib/hadoop-yarn/nm_security_enabled',
+                              content= 'Marker file to track first start after enabling/disabling
security. During first start yarn local, log dirs are removed and recreated'
+    )
+    self.assertResourceCalled('Directory', '/hadoop/yarn/local',
                               owner = 'yarn',
                               group = 'hadoop',
                               recursive = True,
@@ -350,10 +350,6 @@ class TestNodeManager(RMFTestCase):
                               cd_access='a'
                               )
 
-    self.assertResourceCalled('Execute', ('chown', '-R', u'ambari-qa', u'/hadoop/yarn/local/usercache/ambari-qa'),
-        sudo = True,
-        only_if = 'test -d /hadoop/yarn/local/usercache/ambari-qa',
-    )
     self.assertResourceCalled('Directory', '/var/run/hadoop-yarn',
       owner = 'yarn',
       group = 'hadoop',

http://git-wip-us.apache.org/repos/asf/ambari/blob/a79d5ae3/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
index 428655b..fb7d847 100644
--- a/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
+++ b/ambari-server/src/test/python/stacks/2.0.6/YARN/test_resourcemanager.py
@@ -434,6 +434,12 @@ class TestResourceManager(RMFTestCase):
       owner = 'yarn',
       group = 'hadoop',
     )
+    self.assertResourceCalled('Execute', 'yarn resourcemanager -format-state-store',
+                              user = 'yarn',
+    )
+    self.assertResourceCalled('File', '/var/lib/hadoop-yarn/rm_security_enabled',
+                              content = 'Marker file to track first start after enabling/disabling
security. During first start ResourceManager state store is formatted',
+    )
     self.assertResourceCalled('File', '/etc/hadoop/conf/yarn.exclude',
       owner = 'yarn',
       group = 'hadoop',


Mime
View raw message