ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From aonis...@apache.org
Subject ambari git commit: AMBARI-9647. Unable to stop region server on Ambari with security=yes (aonishuk)
Date Sun, 15 Feb 2015 12:21:29 GMT
Repository: ambari
Updated Branches:
  refs/heads/trunk 394471622 -> 97eac81b0


AMBARI-9647. Unable to stop region server on Ambari with security=yes (aonishuk)


Project: http://git-wip-us.apache.org/repos/asf/ambari/repo
Commit: http://git-wip-us.apache.org/repos/asf/ambari/commit/97eac81b
Tree: http://git-wip-us.apache.org/repos/asf/ambari/tree/97eac81b
Diff: http://git-wip-us.apache.org/repos/asf/ambari/diff/97eac81b

Branch: refs/heads/trunk
Commit: 97eac81b0bdfae1293d9fbbe75a80751d3d239e3
Parents: 3944716
Author: Andrew Onishuk <aonishuk@hortonworks.com>
Authored: Sun Feb 15 14:21:22 2015 +0200
Committer: Andrew Onishuk <aonishuk@hortonworks.com>
Committed: Sun Feb 15 14:21:22 2015 +0200

----------------------------------------------------------------------
 .../resource_management/TestExecuteResource.py  | 10 +++--
 .../resource_management/TestGroupResource.py    | 13 +++---
 .../resource_management/TestUserResource.py     | 25 ++++++-----
 .../python/resource_management/core/shell.py    | 47 +++++++++++++-------
 4 files changed, 58 insertions(+), 37 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ambari/blob/97eac81b/ambari-agent/src/test/python/resource_management/TestExecuteResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestExecuteResource.py b/ambari-agent/src/test/python/resource_management/TestExecuteResource.py
index a0b375b..5013bad 100644
--- a/ambari-agent/src/test/python/resource_management/TestExecuteResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestExecuteResource.py
@@ -101,10 +101,12 @@ class TestExecuteResource(TestCase):
   def test_attribute_try_sleep_tries(self, popen_mock, time_mock):
     expected_call = "call('Retrying after %d seconds. Reason: %s', 1, 'Fail')"
 
-    subproc_mock = MagicMock()
-    subproc_mock.returncode = 0
-    subproc_mock.stdout.readline = MagicMock(side_effect = [Fail("Fail"), "OK"])
-    popen_mock.return_value = subproc_mock
+    subproc_mock_one = MagicMock()
+    subproc_mock_one.returncode = 1
+    subproc_mock_zero = MagicMock()
+    subproc_mock_zero.returncode = 0
+    #subproc_mock.stdout.readline = MagicMock(side_effect = [Fail("Fail"), "OK"])
+    popen_mock.side_effect = [subproc_mock_one, subproc_mock_zero]
 
     with Environment("/") as env:
       Execute('echo "1"',

http://git-wip-us.apache.org/repos/asf/ambari/blob/97eac81b/ambari-agent/src/test/python/resource_management/TestGroupResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestGroupResource.py b/ambari-agent/src/test/python/resource_management/TestGroupResource.py
index 7ef487a..850690f 100644
--- a/ambari-agent/src/test/python/resource_management/TestGroupResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestGroupResource.py
@@ -25,10 +25,13 @@ from resource_management.core.system import System
 import subprocess
 import grp
 import os
+import pty
 
 
 @patch.object(System, "os_family", new = 'redhat')
 @patch.object(os, "environ", new = {'PATH':'/bin'})
+@patch.object(pty, "openpty", new = MagicMock(return_value=(1,5)))
+@patch.object(os, "close", new=MagicMock())
 class TestGroupResource(TestCase):
 
   @patch.object(grp, "getgrnam")
@@ -48,7 +51,7 @@ class TestGroupResource(TestCase):
     
 
     self.assertEqual(popen_mock.call_count, 1)
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E groupadd -p secure hadoop"], shell=False, preexec_fn=None, stderr=-2, stdout=-1,
env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E groupadd -p secure hadoop"], shell=False, preexec_fn=None, stderr=-2, stdout=5,
bufsize=1, env={'PATH': '/bin'}, cwd=None)
     getgrnam_mock.assert_called_with('hadoop')
 
 
@@ -70,7 +73,7 @@ class TestGroupResource(TestCase):
     
 
     self.assertEqual(popen_mock.call_count, 1)
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E groupmod -p secure -g 2 mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=-1, env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E groupmod -p secure -g 2 mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=5, bufsize=1, env={'PATH': '/bin'}, cwd=None)
     getgrnam_mock.assert_called_with('mapred')
 
 
@@ -95,7 +98,7 @@ class TestGroupResource(TestCase):
     except Fail:
       pass
     self.assertEqual(popen_mock.call_count, 1)
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E groupmod -p secure -g 2 mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=-1, env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E groupmod -p secure -g 2 mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=5, bufsize=1, env={'PATH': '/bin'}, cwd=None)
     getgrnam_mock.assert_called_with('mapred')
 
 
@@ -116,7 +119,7 @@ class TestGroupResource(TestCase):
     
 
     self.assertEqual(popen_mock.call_count, 1)
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', 'groupdel
mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', 'groupdel
mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=5, bufsize=1, env={'PATH': '/bin'},
cwd=None)
     getgrnam_mock.assert_called_with('mapred')
 
 
@@ -141,5 +144,5 @@ class TestGroupResource(TestCase):
       pass
 
     self.assertEqual(popen_mock.call_count, 1)
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', 'groupdel
mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', 'groupdel
mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=5, bufsize=1, env={'PATH': '/bin'},
cwd=None)
     getgrnam_mock.assert_called_with('mapred')

http://git-wip-us.apache.org/repos/asf/ambari/blob/97eac81b/ambari-agent/src/test/python/resource_management/TestUserResource.py
----------------------------------------------------------------------
diff --git a/ambari-agent/src/test/python/resource_management/TestUserResource.py b/ambari-agent/src/test/python/resource_management/TestUserResource.py
index dad2828..ca7e669 100644
--- a/ambari-agent/src/test/python/resource_management/TestUserResource.py
+++ b/ambari-agent/src/test/python/resource_management/TestUserResource.py
@@ -25,9 +25,12 @@ from resource_management.core.resources import User
 import pwd
 import subprocess
 import os
+import pty
 
 @patch.object(System, "os_family", new = 'redhat')
 @patch.object(os, "environ", new = {'PATH':'/bin'})
+@patch.object(pty, "openpty", new = MagicMock(return_value=(1,5)))
+@patch.object(os, "close", new=MagicMock())
 class TestUserResource(TestCase):
 
   @patch.object(subprocess, "Popen")
@@ -41,7 +44,7 @@ class TestUserResource(TestCase):
     with Environment('/') as env:
       user = User("mapred", action = "create", shell = "/bin/bash")
 
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E useradd -m -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=-1, env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E useradd -m -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=5, env={'PATH': '/bin'}, bufsize=1, cwd=None)
     self.assertEqual(popen_mock.call_count, 1)
 
   @patch.object(subprocess, "Popen")
@@ -56,7 +59,7 @@ class TestUserResource(TestCase):
     with Environment('/') as env:
       user = User("mapred", action = "create", shell = "/bin/bash")
 
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1,
env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=5,
bufsize=1, env={'PATH': '/bin'}, cwd=None)
     self.assertEqual(popen_mock.call_count, 1)
 
   @patch.object(subprocess, "Popen")
@@ -71,7 +74,7 @@ class TestUserResource(TestCase):
     with Environment('/') as env:
       user = User("mapred", action = "remove", shell = "/bin/bash")
 
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', 'userdel
mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=-1, env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', 'userdel
mapred'], shell=False, preexec_fn=None, stderr=-2, stdout=5, bufsize=1, env={'PATH': '/bin'},
cwd=None)
     self.assertEqual(popen_mock.call_count, 1)
 
   @patch.object(subprocess, "Popen")
@@ -87,7 +90,7 @@ class TestUserResource(TestCase):
       user = User("mapred", action = "create", comment = "testComment", 
           shell = "/bin/bash")
 
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -c testComment -s /bin/bash mapred"], shell=False, preexec_fn=None,
stderr=-2, stdout=-1, env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -c testComment -s /bin/bash mapred"], shell=False, preexec_fn=None,
stderr=-2, stdout=5, bufsize=1, env={'PATH': '/bin'}, cwd=None)
     self.assertEqual(popen_mock.call_count, 1)
 
   @patch.object(subprocess, "Popen")
@@ -103,7 +106,7 @@ class TestUserResource(TestCase):
       user = User("mapred", action = "create", home = "/test/home", 
           shell = "/bin/bash")
 
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -s /bin/bash -d /test/home mapred"], shell=False, preexec_fn=None,
stderr=-2, stdout=-1, env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -s /bin/bash -d /test/home mapred"], shell=False, preexec_fn=None,
stderr=-2, stdout=5, bufsize=1, env={'PATH': '/bin'}, cwd=None)
     self.assertEqual(popen_mock.call_count, 1)
 
   @patch.object(subprocess, "Popen")
@@ -119,7 +122,7 @@ class TestUserResource(TestCase):
       user = User("mapred", action = "create", password = "secure", 
           shell = "/bin/bash")    
 
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -s /bin/bash -p secure mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=-1, env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -s /bin/bash -p secure mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=5, bufsize=1, env={'PATH': '/bin'}, cwd=None)
     self.assertEqual(popen_mock.call_count, 1)
 
   @patch.object(subprocess, "Popen")
@@ -134,7 +137,7 @@ class TestUserResource(TestCase):
     with Environment('/') as env:
       user = User("mapred", action = "create", shell = "/bin/sh")
 
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -s /bin/sh mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1,
env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -s /bin/sh mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=5,
bufsize=1, env={'PATH': '/bin'}, cwd=None)
     self.assertEqual(popen_mock.call_count, 1)
 
   @patch.object(subprocess, "Popen")
@@ -149,7 +152,7 @@ class TestUserResource(TestCase):
     with Environment('/') as env:
       user = User("mapred", action = "create", uid = "1", shell = "/bin/bash")
 
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -s /bin/bash -u 1 mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=-1, env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -s /bin/bash -u 1 mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=5, bufsize=1, env={'PATH': '/bin'}, cwd=None)
     self.assertEqual(popen_mock.call_count, 1)
 
   @patch.object(subprocess, "Popen")
@@ -164,7 +167,7 @@ class TestUserResource(TestCase):
     with Environment('/') as env:
       user = User("mapred", action = "create", gid = "1", shell = "/bin/bash")
 
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -s /bin/bash -g 1 mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=-1, env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -s /bin/bash -g 1 mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=5, bufsize=1, env={'PATH': '/bin'}, cwd=None)
     self.assertEqual(popen_mock.call_count, 1)
 
   @patch.object(subprocess, "Popen")
@@ -180,7 +183,7 @@ class TestUserResource(TestCase):
       user = User("mapred", action = "create", groups = ['1','2','3'], 
           shell = "/bin/bash")
 
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -G 1,2,3 -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=-1, env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E usermod -G 1,2,3 -s /bin/bash mapred"], shell=False, preexec_fn=None, stderr=-2,
stdout=5, bufsize=1, env={'PATH': '/bin'}, cwd=None)
     self.assertEqual(popen_mock.call_count, 1)
 
   @patch.object(subprocess, "Popen")
@@ -194,6 +197,6 @@ class TestUserResource(TestCase):
     with Environment('/') as env:
       user = User("mapred", action = "create")
 
-    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E useradd -m mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=-1,
env={'PATH': '/bin'}, cwd=None)
+    popen_mock.assert_called_with(['/bin/bash', '--login', '--noprofile', '-c', "/usr/bin/sudo
 PATH=/bin -H -E useradd -m mapred"], shell=False, preexec_fn=None, stderr=-2, stdout=5, bufsize=1,
env={'PATH': '/bin'}, cwd=None)
     self.assertEqual(popen_mock.call_count, 1)
 

http://git-wip-us.apache.org/repos/asf/ambari/blob/97eac81b/ambari-common/src/main/python/resource_management/core/shell.py
----------------------------------------------------------------------
diff --git a/ambari-common/src/main/python/resource_management/core/shell.py b/ambari-common/src/main/python/resource_management/core/shell.py
index 3301481..956ba01 100644
--- a/ambari-common/src/main/python/resource_management/core/shell.py
+++ b/ambari-common/src/main/python/resource_management/core/shell.py
@@ -19,10 +19,12 @@ limitations under the License.
 Ambari Agent
 
 """
-import os
 
 __all__ = ["non_blocking_call", "checked_call", "call", "quote_bash_args", "as_user", "as_sudo"]
 
+import os
+import pty
+import select
 import sys
 import logging
 import string
@@ -134,9 +136,10 @@ def _call(command, logoutput=None, throw_on_failure=True,
   for placeholder, replacement in PLACEHOLDERS_TO_STR.iteritems():
     command = command.replace(placeholder, replacement.format(env_str=env_str))
 
+  master_fd, slave_fd = pty.openpty()
   # --noprofile is used to preserve PATH set for ambari-agent
   subprocess_command = ["/bin/bash","--login","--noprofile","-c", command]
-  proc = subprocess.Popen(subprocess_command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+  proc = subprocess.Popen(subprocess_command, bufsize=1, stdout=slave_fd, stderr=subprocess.STDOUT,
                           cwd=cwd, env=env, shell=False,
                           preexec_fn=preexec_fn)
   
@@ -151,23 +154,32 @@ def _call(command, logoutput=None, throw_on_failure=True,
   # in case logoutput==False, never log.    
   logoutput = logoutput==True and Logger.logger.isEnabledFor(logging.INFO) or logoutput==None
and Logger.logger.isEnabledFor(logging.DEBUG)
   out = ""
-  
+  read_timeout = .04 # seconds
+
   try:
-    for line in iter(proc.stdout.readline, b''):
-      out += line
-      
-      try:
-        if on_new_line:
-          on_new_line(line)
-      except Exception, err:
-        err_msg = "Caused by on_new_line function failed with exception for input argument
'{0}':\n{1}".format(line, traceback.format_exc())
-        raise Fail(err_msg)
-        
-      if logoutput:
-        _print(line)
+    while True:
+      ready, _, _ = select.select([master_fd], [], [], read_timeout)
+      if ready:
+        line = os.read(master_fd, 512)
+        if not line:
+            break
+          
+        out += line
+        try:
+          if on_new_line:
+            on_new_line(line)
+        except Exception, err:
+          err_msg = "Caused by on_new_line function failed with exception for input argument
'{0}':\n{1}".format(line, traceback.format_exc())
+          raise Fail(err_msg)
+          
+        if logoutput:
+          _print(line)    
+      elif proc.poll() is not None:
+        break # proc exited
   finally:
-    proc.stdout.close()
-    
+    os.close(slave_fd)
+    os.close(master_fd)
+
   proc.wait()  
   out = out.strip('\n')
   
@@ -179,6 +191,7 @@ def _call(command, logoutput=None, throw_on_failure=True,
       raise ExecuteTimeoutException()
    
   code = proc.returncode
+  print code
   
   if throw_on_failure and code:
     err_msg = Logger.filter_text(("Execution of '%s' returned %d. %s") % (command_alias,
code, out))


Mime
View raw message