ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From jonathanhur...@apache.org
Subject [45/50] ambari git commit: AMBARI-20670 Node manager start extremely slow when YARN NM local dirs are very large (dgrinenko via dsen)
Date Thu, 06 Apr 2017 18:58:34 GMT
AMBARI-20670 Node manager start extremely slow when YARN NM local dirs are very large (dgrinenko
via dsen)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/28238ae8
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/28238ae8
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/28238ae8

Branch: refs/heads/branch-feature-AMBARI-12556
Commit: 28238ae84e24aa72613e59b6af341f1c2452a0aa
Parents: c11d004
Author: Dmytro Sen <dsen@apache.org>
Authored: Thu Apr 6 15:22:10 2017 +0300
Committer: Dmytro Sen <dsen@apache.org>
Committed: Thu Apr 6 15:22:10 2017 +0300

----------------------------------------------------------------------
 .../python/resource_management/TestUtils.py     | 39 +++++++++++++
 .../python/resource_management/core/sudo.py     | 58 +++++++++++++++-----
 .../python/resource_management/core/utils.py    | 56 ++++++++++++++++++-
 .../YARN/2.1.0.2.0/package/scripts/yarn.py      | 16 ++++--
 .../YARN/3.0.0.3.0/package/scripts/yarn.py      | 14 +++--
 5 files changed, 159 insertions(+), 24 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-agent/src/test/python/resource_management/TestUtils.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestUtils.py b/ambari-agent/src/test/python/resource_management/TestUtils.py
new file mode 100644
index 0000000..0fbae3e
--- /dev/null
+++ b/ambari-agent/src/test/python/resource_management/TestUtils.py
@@ -0,0 +1,39 @@
+"""
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+"""
+
+import stat
+from unittest import TestCase
+from resource_management.core.utils import attr_to_bitmask
+
+
+class TestUtils(TestCase):
+
+  def test_attr_to_bitmask(self):
+    test_set = [
+      ["+r", stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH, 0],
+      ["u+w", stat.S_IWUSR, 0],
+      ["uo+x", stat.S_IXUSR | stat.S_IXOTH, 0],
+      ["-x", stat.S_IRUSR, stat.S_IXUSR | stat.S_IXOTH | stat.S_IRUSR],
+      ["=x", stat.S_IXUSR | stat.S_IXOTH | stat.S_IXGRP, stat.S_IRUSR | stat.S_IRGRP]
+    ]
+
+    for test in test_set:
+      test_pattern, expected, initial_val = test
+      bitmask = attr_to_bitmask(test_pattern, initial_bitmask= initial_val)
+      self.assertEquals(expected, bitmask, "Test set \"{0}\" failed, expected: {1} but got
{2}".format(
+        test_pattern, expected, bitmask))

http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-common/src/main/python/resource_management/core/sudo.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/sudo.py b/ambari-common/src/main/python/resource_management/core/sudo.py
index d6fd71f..c350161 100644
--- a/ambari-common/src/main/python/resource_management/core/sudo.py
+++ b/ambari-common/src/main/python/resource_management/core/sudo.py
@@ -30,6 +30,7 @@ from resource_management.core import shell
 from resource_management.core.exceptions import Fail
 import subprocess
 
+from resource_management.core.utils import attr_to_bitmask
 
 if os.geteuid() == 0:
   def chown(path, owner, group):
@@ -54,16 +55,45 @@ if os.geteuid() == 0:
             
   
   def chmod(path, mode):
+    """
+    Wrapper around python function
+    
+    :type path str
+    :type mode int
+    """
     return os.chmod(path, mode)
   
-  mode_to_stat = {"a+x": stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH, "a+rx": stat.S_IXUSR
| stat.S_IXGRP | stat.S_IXOTH | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH, "u+x": stat.S_IXUSR,
"g+x": stat.S_IXGRP,  "o+x": stat.S_IXOTH}
+
   def chmod_extended(path, mode):
-    if mode in mode_to_stat:
-      st = os.stat(path)
-      os.chmod(path, st.st_mode | mode_to_stat[mode])
-    else:
-      shell.checked_call(["chmod", mode, path])
-      
+    """
+    :type path str
+    :type mode str
+    """
+    st = os.stat(path)
+    os.chmod(path, attr_to_bitmask(mode, initial_bitmask=st.st_mode))
+
+  def chmod_recursive(path, recursive_mode_flags, recursion_follow_links=False):
+    """
+    Change recursively permissions on directories or files
+    
+    :type path str
+    :type recursive_mode_flags
+    :type recursion_follow_links bool
+    """
+    dir_attrib = recursive_mode_flags["d"] if "d" in recursive_mode_flags else None
+    files_attrib = recursive_mode_flags["f"] if "d" in recursive_mode_flags else None
+
+    for root, dirs, files in os.walk(path, followlinks=recursion_follow_links):
+      if dir_attrib is not None:
+        for dir_name in dirs:
+          full_dir_path = os.path.join(root, dir_name)
+          chmod(full_dir_path, attr_to_bitmask(dir_attrib, initial_bitmask=os.stat(full_dir_path).st_mode))
+
+      if files_attrib is not None:
+        for file_name in files:
+          full_file_path = os.path.join(root, file_name)
+          chmod(full_file_path, attr_to_bitmask(files_attrib, initial_bitmask=os.stat(full_file_path).st_mode))
+
   def copy(src, dst):
     shutil.copy(src, dst)
     
@@ -278,10 +308,10 @@ else:
     return files
 
 
-def chmod_recursive(path, recursive_mode_flags, recursion_follow_links):
-  find_flags = []
-  if recursion_follow_links:
-    find_flags.append('-L')
-    
-  for key, flags in recursive_mode_flags.iteritems():
-    shell.checked_call(["find"] + find_flags + [path, "-type", key, "-exec" , "chmod", flags
,"{}" ,";"])
+  def chmod_recursive(path, recursive_mode_flags, recursion_follow_links):
+    find_flags = []
+    if recursion_follow_links:
+      find_flags.append('-L')
+
+    for key, flags in recursive_mode_flags.iteritems():
+      shell.checked_call(["find"] + find_flags + [path, "-type", key, "-exec" , "chmod",
flags ,"{}" ,";"])

http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-common/src/main/python/resource_management/core/utils.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/utils.py b/ambari-common/src/main/python/resource_management/core/utils.py
index 265b2f2..53f6b8b 100644
--- a/ambari-common/src/main/python/resource_management/core/utils.py
+++ b/ambari-common/src/main/python/resource_management/core/utils.py
@@ -27,10 +27,16 @@ import sys
 import signal
 import cStringIO
 from functools import wraps
+
+import re
+
 from resource_management.core.exceptions import Fail
 from itertools import chain, repeat, islice
 
 PASSWORDS_HIDE_STRING = "[PROTECTED]"
+PERM_STRING_REGEXP = re.compile("(?P<scope>[ugoa]*)(?P<direction>[-+=])(?P<attr>[rwx]*)")
+PERM_REGISTER = {"u": 0o100, "g": 0o010, "o": 0o001}
+PERM_BITS = {"r": 0o004, "w": 0o002, "x": 0o001}
 
 class AttributeDictionary(object):
   def __init__(self, *args, **kwargs):
@@ -157,4 +163,52 @@ def pad_infinite(iterable, padding=None):
   return chain(iterable, repeat(padding))
 
 def pad(iterable, size, padding=None):
-  return islice(pad_infinite(iterable, padding), size)
\ No newline at end of file
+  return islice(pad_infinite(iterable, padding), size)
+
+
+def attr_to_bitmask(attr, initial_bitmask=0o0):
+  """
+  Function able to generate permission bits from passed named permission string (chmod like
style)
+   
+  Supports:
+   - scope modifications: u,g,o or a 
+   - setting mode: +,-,-
+   - attributes: r,x,w
+   
+  Samples:
+    uo+rw, a+x, u-w, o=r
+  
+  :type attr str 
+  :type initial_bitmask int
+  """
+  attr_dict = {"scope": "", "direction": "", "attr": ""}
+  re_match_result = PERM_STRING_REGEXP.match(attr)
+
+  if re_match_result:
+    attr_dict = re_match_result.groupdict(default=attr_dict)
+
+  if attr_dict["scope"] == "":
+    attr_dict["scope"] = "a"
+
+  if "a" in attr_dict["scope"]:
+    attr_dict["scope"] = "ugo"
+
+  attr_dict["scope"] = list(attr_dict["scope"])
+  attr_dict["attr"] = list(attr_dict["attr"])
+
+  if attr_dict["direction"] == "=":
+    clear_mask = 0o0
+    for scope in attr_dict["scope"]:
+      clear_mask = clear_mask | 0o007 * PERM_REGISTER[scope]
+
+    initial_bitmask = initial_bitmask ^ (initial_bitmask & clear_mask)
+    attr_dict["direction"] = "+"
+
+  for scope in attr_dict["scope"]:
+    for attr in attr_dict["attr"]:
+      if attr_dict["direction"] == "-" and (initial_bitmask & (PERM_BITS[attr] * PERM_REGISTER[scope]))
> 0:
+        initial_bitmask = initial_bitmask ^ (PERM_BITS[attr] * PERM_REGISTER[scope])
+      elif attr_dict["direction"] == "+":
+        initial_bitmask = initial_bitmask | (PERM_BITS[attr] * PERM_REGISTER[scope])
+
+  return initial_bitmask

http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
index 204ab56..52338df 100644
--- a/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/2.1.0.2.0/package/scripts/yarn.py
@@ -347,7 +347,6 @@ def setup_nodemanager():
     elif not params.security_enabled:
       File(params.nm_security_marker, action="delete")
 
-
   if not params.security_enabled or params.toggle_nm_security:
     # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable
(unmounted), and intended to reside on a different mount.
     nm_log_dir_to_mount_file_content = handle_mounted_dirs(create_log_dir, params.nm_log_dirs,
params.nm_log_dir_to_mount_file, params)
@@ -459,17 +458,24 @@ def create_log_dir(dir_name):
             ignore_failures=True,
   )
 
+
 def create_local_dir(dir_name):
   import params
+
+  directory_args = {}
+
+  if params.toggle_nm_security:
+    directory_args["recursive_mode_flags"] = {'f': 'a+rw', 'd': 'a+rwx'}
+
   Directory(dir_name,
-            create_parents = True,
+            create_parents=True,
             cd_access="a",
             mode=0755,
             owner=params.yarn_user,
             group=params.user_group,
             ignore_failures=True,
-            recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'},
-  )
+            **directory_args
+            )
 
 @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
 def yarn(name = None):
@@ -500,4 +506,4 @@ def yarn(name = None):
     ServiceConfig(service_name,
                   action="change_user",
                   username = params.yarn_user,
-                  password = Script.get_password(params.yarn_user))
\ No newline at end of file
+                  password = Script.get_password(params.yarn_user))

http://git-wip-us.apache.org/repos/asf/ambari/blob/28238ae8/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
----------------------------------------------------------------------
diff --git a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
index ae1b425..d601f8f 100644
--- a/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
+++ b/ambari-server/src/main/resources/common-services/YARN/3.0.0.3.0/package/scripts/yarn.py
@@ -347,7 +347,6 @@ def setup_nodemanager():
     elif not params.security_enabled:
       File(params.nm_security_marker, action="delete")
 
-
   if not params.security_enabled or params.toggle_nm_security:
     # handle_mounted_dirs ensures that we don't create dirs which are temporary unavailable
(unmounted), and intended to reside on a different mount.
     nm_log_dir_to_mount_file_content = handle_mounted_dirs(create_log_dir, params.nm_log_dirs,
params.nm_log_dir_to_mount_file, params)
@@ -459,16 +458,23 @@ def create_log_dir(dir_name):
             ignore_failures=True,
   )
 
+
 def create_local_dir(dir_name):
   import params
+
+  directory_args = {}
+
+  if params.toggle_nm_security:
+    directory_args["recursive_mode_flags"] = {'f': 'a+rw', 'd': 'a+rwx'}
+
   Directory(dir_name,
-            create_parents = True,
+            create_parents=True,
             cd_access="a",
             mode=0755,
             owner=params.yarn_user,
             group=params.user_group,
             ignore_failures=True,
-            recursive_mode_flags = {'f': 'a+rw', 'd': 'a+rwx'},
+            **directory_args
   )
 
 @OsFamilyFuncImpl(os_family=OSConst.WINSRV_FAMILY)
@@ -500,4 +506,4 @@ def yarn(name = None):
     ServiceConfig(service_name,
                   action="change_user",
                   username = params.yarn_user,
-                  password = Script.get_password(params.yarn_user))
\ No newline at end of file
+                  password = Script.get_password(params.yarn_user))


Mime
View raw message