Return-Path: X-Original-To: apmail-incubator-ambari-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-ambari-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 01FA8F863 for ; Sat, 13 Apr 2013 00:46:50 +0000 (UTC) Received: (qmail 90909 invoked by uid 500); 13 Apr 2013 00:46:49 -0000 Delivered-To: apmail-incubator-ambari-commits-archive@incubator.apache.org Received: (qmail 90869 invoked by uid 500); 13 Apr 2013 00:46:49 -0000 Mailing-List: contact ambari-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@incubator.apache.org Delivered-To: mailing list ambari-commits@incubator.apache.org Received: (qmail 90859 invoked by uid 99); 13 Apr 2013 00:46:49 -0000 Received: from nike.apache.org (HELO nike.apache.org) (192.87.106.230) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 13 Apr 2013 00:46:49 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Sat, 13 Apr 2013 00:46:44 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 2B0F823888E4; Sat, 13 Apr 2013 00:46:22 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1467531 - in /incubator/ambari/trunk: ./ ambari-server/src/main/java/org/apache/ambari/server/bootstrap/ ambari-server/src/main/python/ ambari-server/src/test/java/org/apache/ambari/server/bootstrap/ ambari-server/src/test/python/ Date: Sat, 13 Apr 2013 00:46:21 -0000 To: ambari-commits@incubator.apache.org From: swagle@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130413004622.2B0F823888E4@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: swagle Date: Sat Apr 13 00:46:21 2013 New Revision: 1467531 URL: http://svn.apache.org/r1467531 Log: AMBARI-1922. Support not root ssh via a user that can sudo in as root. (Sumit Mohanty via swagle) Modified: incubator/ambari/trunk/CHANGES.txt incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSRunner.java incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/SshHostInfo.java incubator/ambari/trunk/ambari-server/src/main/python/bootstrap.py incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/bootstrap/BootStrapTest.java incubator/ambari/trunk/ambari-server/src/test/python/TestBootstrap.py Modified: incubator/ambari/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/CHANGES.txt?rev=1467531&r1=1467530&r2=1467531&view=diff ============================================================================== --- incubator/ambari/trunk/CHANGES.txt (original) +++ incubator/ambari/trunk/CHANGES.txt Sat Apr 13 00:46:21 2013 @@ -11,6 +11,9 @@ Trunk (unreleased changes): INCOMPATIBLE CHANGES NEW FEATURES + + AMBARI-1922. Support not root ssh via a user that can sudo in as root. + (Sumit Mohanty via swagle) AMBARI-1914. Add Nagios alerts for Hue service. (swagle) Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSRunner.java URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSRunner.java?rev=1467531&r1=1467530&r2=1467531&view=diff ============================================================================== --- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSRunner.java (original) +++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/BSRunner.java Sat Apr 13 00:46:21 2013 @@ -39,12 +39,15 @@ import org.apache.commons.logging.LogFac class BSRunner extends Thread { private static Log LOG = LogFactory.getLog(BSRunner.class); + private static final String DEFAULT_USER = "root"; + private boolean finished = false; private SshHostInfo sshHostInfo; private File bootDir; private String bsScript; private File requestIdDir; private File sshKeyFile; + private File passwordFile; private int requestId; private String agentSetupScript; private String agentSetupPassword; @@ -130,6 +133,10 @@ class BSRunner extends Thread { FileUtils.writeStringToFile(sshKeyFile, data); } + private void writePasswordFile(String data) throws IOException { + FileUtils.writeStringToFile(passwordFile, data); + } + public synchronized void finished() { this.finished = true; } @@ -137,7 +144,11 @@ class BSRunner extends Thread { @Override public void run() { String hostString = createHostString(sshHostInfo.getHosts()); - String commands[] = new String[6]; + String user = sshHostInfo.getUser(); + if (user == null || user.isEmpty()) { + user = DEFAULT_USER; + } + String commands[] = new String[8]; String shellCommand[] = new String[3]; BSStat stat = BSStat.RUNNING; String scriptlog = ""; @@ -150,9 +161,19 @@ class BSRunner extends Thread { + sshHostInfo.getSshKey() + "\""); } + String password = sshHostInfo.getPassword(); + if (password != null && !password.isEmpty()) { + this.passwordFile = new File(this.requestIdDir, "host_pass"); + // TODO : line separator should be changed + // if we are going to support multi platform server-agent solution + String lineSeparator = System.getProperty("line.separator"); + password = password + lineSeparator; + writePasswordFile(password); + } + writeSshKeyFile(sshHostInfo.getSshKey()); /* Running command: - * script hostlist bsdir sshkeyfile + * script hostlist bsdir user sshkeyfile */ shellCommand[0] = "sh"; shellCommand[1] = "-c"; @@ -160,11 +181,16 @@ class BSRunner extends Thread { commands[0] = this.bsScript; commands[1] = hostString; commands[2] = this.requestIdDir.toString(); - commands[3] = this.sshKeyFile.toString(); - commands[4] = this.agentSetupScript.toString(); - commands[5] = this.ambariHostname; + commands[3] = user; + commands[4] = this.sshKeyFile.toString(); + commands[5] = this.agentSetupScript.toString(); + commands[6] = this.ambariHostname; + if (this.passwordFile != null) { + commands[7] = this.passwordFile.toString(); + } LOG.info("Host= " + hostString + " bs=" + this.bsScript + " requestDir=" + - requestIdDir + " keyfile=" + this.sshKeyFile + " server=" + this.ambariHostname); + requestIdDir + " user=" + user + " keyfile=" + this.sshKeyFile + + " passwordFile " + this.passwordFile + " server=" + this.ambariHostname); String[] env = new String[] { "AMBARI_PASSPHRASE=" + agentSetupPassword }; if (this.verbose) @@ -275,7 +301,15 @@ class BSRunner extends Thread { try { FileUtils.forceDelete(sshKeyFile); } catch (IOException io) { - LOG.info(io.getMessage()); + LOG.warn(io.getMessage()); + } + if (passwordFile != null) { + // Remove password file after bootstrap is complete + try { + FileUtils.forceDelete(passwordFile); + } catch (IOException io) { + LOG.warn(io.getMessage()); + } } finished(); } Modified: incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/SshHostInfo.java URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/SshHostInfo.java?rev=1467531&r1=1467530&r2=1467531&view=diff ============================================================================== --- incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/SshHostInfo.java (original) +++ incubator/ambari/trunk/ambari-server/src/main/java/org/apache/ambari/server/bootstrap/SshHostInfo.java Sat Apr 13 00:46:21 2013 @@ -41,10 +41,16 @@ public class SshHostInfo { @XmlElement private List hosts = new ArrayList(); - + @XmlElement private boolean verbose = false; + @XmlElement + private String user; + + @XmlElement + private String password; + public String getSshKey() { return sshKey; } @@ -69,6 +75,22 @@ public class SshHostInfo { this.verbose = verbose; } + public String getUser() { + return user; + } + + public void setUser(String user) { + this.user = user; + } + + public String getPassword() { + return password; + } + + public void setPassword(String password) { + this.password = password; + } + public String hostListAsString() { StringBuilder ret = new StringBuilder(); if (this.hosts == null) { Modified: incubator/ambari/trunk/ambari-server/src/main/python/bootstrap.py URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/main/python/bootstrap.py?rev=1467531&r1=1467530&r2=1467531&view=diff ============================================================================== --- incubator/ambari/trunk/ambari-server/src/main/python/bootstrap.py (original) +++ incubator/ambari/trunk/ambari-server/src/main/python/bootstrap.py Sat Apr 13 00:46:21 2013 @@ -35,7 +35,8 @@ HOST_BOOTSTRAP_TIMEOUT = 300 class SCP(threading.Thread): """ SCP implementation that is thread based. The status can be returned using status val """ - def __init__(self, sshKeyFile, host, inputFile, remote, bootdir): + def __init__(self, user, sshKeyFile, host, inputFile, remote, bootdir): + self.user = user self.sshKeyFile = sshKeyFile self.host = host self.inputFile = inputFile @@ -58,7 +59,7 @@ class SCP(threading.Thread): "-o", "ConnectTimeout=60", "-o", "BatchMode=yes", "-o", "StrictHostKeyChecking=no", - "-i", self.sshKeyFile, self.inputFile, "root@" + + "-i", self.sshKeyFile, self.inputFile, self.user + "@" + self.host + ":" + self.remote] logging.info("Running scp command " + ' '.join(scpcommand)) scpstat = subprocess.Popen(scpcommand, stdout=subprocess.PIPE, @@ -67,19 +68,25 @@ class SCP(threading.Thread): self.ret["exitstatus"] = scpstat.returncode self.ret["log"] = "STDOUT\n" + log[0] + "\nSTDERR\n" + log[1] logFilePath = os.path.join(self.bootdir, self.host + ".log") + self.writeLogToFile(logFilePath) + logging.info("scp " + self.inputFile + " done for host " + self.host + ", exitcode=" + str(scpstat.returncode)) + pass + + def writeLogToFile(self, logFilePath): logFile = open(logFilePath, "a+") logFile.write(self.ret["log"]) logFile.close - logging.info("scp " + self.inputFile + " done for host " + self.host + ", exitcode=" + str(scpstat.returncode)) pass class SSH(threading.Thread): """ Ssh implementation of this """ - def __init__(self, sshKeyFile, host, command, bootdir): + def __init__(self, user, sshKeyFile, host, command, bootdir, errorMessage = None): + self.user = user self.sshKeyFile = sshKeyFile self.host = host self.command = command self.bootdir = bootdir + self.errorMessage = errorMessage self.ret = {"exitstatus" : -1, "log": "FAILED"} threading.Thread.__init__(self) self.daemon = True @@ -98,24 +105,35 @@ class SSH(threading.Thread): "-o", "BatchMode=yes", "-tt", # Should prevent "tput: No value for $TERM and no -T specified" warning "-i", self.sshKeyFile, - "root@" + self.host, self.command] + self.user + "@" + self.host, self.command] logging.info("Running ssh command " + ' '.join(sshcommand)) sshstat = subprocess.Popen(sshcommand, stdout=subprocess.PIPE, stderr=subprocess.PIPE) log = sshstat.communicate() self.ret["exitstatus"] = sshstat.returncode - self.ret["log"] = "STDOUT\n" + log[0] + "\nSTDERR\n" + log[1] + errorMsg = log[1] + if self.errorMessage and sshstat.returncode != 0: + errorMsg = self.errorMessage + "\n" + errorMsg + self.ret["log"] = "STDOUT\n" + log[0] + "\nSTDERR\n" + errorMsg logFilePath = os.path.join(self.bootdir, self.host + ".log") + self.writeLogToFile(logFilePath) + + doneFilePath = os.path.join(self.bootdir, self.host + ".done") + self.writeDoneToFile(doneFilePath, str(sshstat.returncode)) + + logging.info("Setup agent done for host " + self.host + ", exitcode=" + str(sshstat.returncode)) + pass + + def writeLogToFile(self, logFilePath): logFile = open(logFilePath, "a+") logFile.write(self.ret["log"]) logFile.close + pass - doneFilePath = os.path.join(self.bootdir, self.host + ".done") + def writeDoneToFile(self, doneFilePath, returncode): doneFile = open(doneFilePath, "w+") - doneFile.write(str(sshstat.returncode)) + doneFile.write(str(returncode)) doneFile.close() - - logging.info("Setup agent done for host " + self.host + ", exitcode=" + str(sshstat.returncode)) pass pass @@ -154,11 +172,13 @@ def get_difference(list1, list2): class PSSH: """Run SSH in parallel for a given list of hosts""" - def __init__(self, hosts, sshKeyFile, command, bootdir): + def __init__(self, hosts, user, sshKeyFile, command, bootdir, errorMessage = None): self.hosts = hosts + self.user = user self.sshKeyFile = sshKeyFile self.command = command self.bootdir = bootdir + self.errorMessage = errorMessage self.ret = {} pass @@ -171,7 +191,7 @@ class PSSH: for chunk in splitlist(self.hosts, 20): chunkstats = [] for host in chunk: - ssh = SSH(self.sshKeyFile, host, self.command, self.bootdir) + ssh = SSH(self.user, self.sshKeyFile, host, self.command, self.bootdir, self.errorMessage) ssh.start() chunkstats.append(ssh) pass @@ -191,8 +211,9 @@ pass class PSCP: """Run SCP in parallel for a given list of hosts""" - def __init__(self, hosts, sshKeyFile, inputfile, remote, bootdir): + def __init__(self, hosts, user, sshKeyFile, inputfile, remote, bootdir): self.hosts = hosts + self.user = user self.sshKeyFile = sshKeyFile self.inputfile = inputfile self.remote = remote @@ -209,7 +230,7 @@ class PSCP: for chunk in splitlist(self.hosts, 20): chunkstats = [] for host in chunk: - scp = SCP(self.sshKeyFile, host, self.inputfile, self.remote, self.bootdir) + scp = SCP(self.user, self.sshKeyFile, host, self.inputfile, self.remote, self.bootdir) scp.start() chunkstats.append(scp) pass @@ -230,14 +251,17 @@ pass class BootStrap: """ BootStrapping the agents on a list of hosts""" - def __init__(self, hosts, sshkeyFile, scriptDir, boottmpdir, setupAgentFile, ambariServer): + def __init__(self, hosts, user, sshkeyFile, scriptDir, boottmpdir, setupAgentFile, ambariServer, passwordFile = None): self.hostlist = hosts self.successive_hostlist = hosts + self.hostlist_to_remove_password_file = None + self.user = user self.sshkeyFile = sshkeyFile self.bootdir = boottmpdir self.scriptDir = scriptDir self.setupAgentFile = setupAgentFile self.ambariServer = ambariServer + self.passwordFile = passwordFile self.statuses = None pass @@ -261,22 +285,57 @@ class BootStrap: def getSetupScript(self): return os.path.join(self.scriptDir, "setupAgent.py") + def getPasswordFile(self): + return "/tmp/host_pass" + + def hasPassword(self): + return self.passwordFile != None and self.passwordFile != 'null' + + def getMoveRepoFileWithPasswordCommand(self, targetDir): + return "sudo -S mv /tmp/ambari.repo " + targetDir + " < " + self.getPasswordFile() + + def getMoveRepoFileWithoutPasswordCommand(self, targetDir): + return "sudo mv /tmp/ambari.repo " + targetDir + + def getMoveRepoFileCommand(self, targetDir): + if self.hasPassword(): + return self.getMoveRepoFileWithPasswordCommand(targetDir) + else: + return self.getMoveRepoFileWithoutPasswordCommand(targetDir) + def copyNeededFiles(self): try: # Copying the files fileToCopy = self.getRepoFile() - targetDir = self.getRepoDir() - pscp = PSCP(self.hostlist, self.sshkeyFile, fileToCopy, targetDir, self.bootdir) + logging.info("Copying repo file to 'tmp' folder...") + pscp = PSCP(self.successive_hostlist, self.user, self.sshkeyFile, fileToCopy, "/tmp", self.bootdir) pscp.run() out = pscp.getstatus() # Prepearing report about failed hosts + failed_current = get_difference(self.successive_hostlist, skip_failed_hosts(out)) self.successive_hostlist = skip_failed_hosts(out) failed = get_difference(self.hostlist, self.successive_hostlist) - logging.info("Parallel scp returns for repo file. Failed hosts are: " + str(failed)) + logging.info("Parallel scp returns for copying repo file. All failed hosts are: " + str(failed) + + ". Failed on last step: " + str(failed_current)) #updating statuses - self.statuses = out + self.statuses = unite_statuses(self.statuses, out) + + logging.info("Moving repo file...") + targetDir = self.getRepoDir() + command = self.getMoveRepoFileCommand(targetDir) + pssh = PSSH(self.successive_hostlist, self.user, self.sshkeyFile, command, self.bootdir) + pssh.run() + out = pssh.getstatus() + # Prepearing report about failed hosts + failed_current = get_difference(self.successive_hostlist, skip_failed_hosts(out)) + self.successive_hostlist = skip_failed_hosts(out) + failed = get_difference(self.hostlist, self.successive_hostlist) + logging.info("Parallel scp returns for moving repo file. All failed hosts are: " + str(failed) + + ". Failed on last step: " + str(failed_current)) + #updating statuses + self.statuses = unite_statuses(self.statuses, out) - pscp = PSCP(self.successive_hostlist, self.sshkeyFile, self.setupAgentFile, "/tmp", self.bootdir) + pscp = PSCP(self.successive_hostlist, self.user, self.sshkeyFile, self.setupAgentFile, "/tmp", self.bootdir) pscp.run() out = pscp.getstatus() # Prepearing report about failed hosts @@ -300,10 +359,22 @@ class BootStrap: pass + def getRunSetupWithPasswordCommand(self): + return "sudo -S python /tmp/setupAgent.py " + os.environ[AMBARI_PASSPHRASE_VAR_NAME] + " " + self.ambariServer + " < " + self.getPasswordFile() + + def getRunSetupWithoutPasswordCommand(self): + return "sudo python /tmp/setupAgent.py " + os.environ[AMBARI_PASSPHRASE_VAR_NAME] + " " + self.ambariServer + + def getRunSetupCommand(self): + if self.hasPassword(): + return self.getRunSetupWithPasswordCommand() + else: + return self.getRunSetupWithoutPasswordCommand() + def runSetupAgent(self): logging.info("Running setup agent...") - command = "python /tmp/setupAgent.py " + os.environ[AMBARI_PASSPHRASE_VAR_NAME] + " " + self.ambariServer - pssh = PSSH(self.successive_hostlist, self.sshkeyFile, command, self.bootdir) + command = self.getRunSetupCommand() + pssh = PSSH(self.successive_hostlist, self.user, self.sshkeyFile, command, self.bootdir) pssh.run() out = pssh.getstatus() @@ -334,13 +405,154 @@ class BootStrap: doneFile.close() pass + def checkSudoPackage(self): + try: + """ Checking 'sudo' package on remote hosts """ + command = "rpm -qa | grep sudo" + pssh = PSSH(self.hostlist, self.user, self.sshkeyFile, command, self.bootdir, "Error: Sudo command is not available. Please install the sudo command.") + pssh.run() + out = pssh.getstatus() + # Prepearing report about failed hosts + self.successive_hostlist = skip_failed_hosts(out) + failed = get_difference(self.hostlist, self.successive_hostlist) + logging.info("Parallel ssh returns for checking 'sudo' package. Failed hosts are: " + str(failed)) + #updating statuses + self.statuses = out + + retstatus = 0 + if not failed: + retstatus = 0 + else: + retstatus = 1 + return retstatus + + except Exception, e: + logging.info("Traceback " + traceback.format_exc()) + pass + pass + + def copyPasswordFile(self): + try: + # Copying the password file + logging.info("Copying password file to 'tmp' folder...") + pscp = PSCP(self.successive_hostlist, self.user, self.sshkeyFile, self.passwordFile, "/tmp", self.bootdir) + pscp.run() + out = pscp.getstatus() + # Prepearing report about failed hosts + failed_current = get_difference(self.successive_hostlist, skip_failed_hosts(out)) + self.successive_hostlist = skip_failed_hosts(out) + self.hostlist_to_remove_password_file = self.successive_hostlist + failed = get_difference(self.hostlist, self.successive_hostlist) + logging.warn("Parallel scp returns for copying password file. All failed hosts are: " + str(failed) + + ". Failed on last step: " + str(failed_current)) + #updating statuses + self.statuses = unite_statuses(self.statuses, out) + + # Change password file mode to 600 + logging.info("Changing password file mode...") + targetDir = self.getRepoDir() + command = "chmod 600 " + self.getPasswordFile() + pssh = PSSH(self.successive_hostlist, self.user, self.sshkeyFile, command, self.bootdir) + pssh.run() + out = pssh.getstatus() + # Prepearing report about failed hosts + failed_current = get_difference(self.successive_hostlist, skip_failed_hosts(out)) + self.successive_hostlist = skip_failed_hosts(out) + failed = get_difference(self.hostlist, self.successive_hostlist) + logging.warning("Parallel scp returns for copying password file. All failed hosts are: " + str(failed) + + ". Failed on last step: " + str(failed_current)) + #updating statuses + self.statuses = unite_statuses(self.statuses, out) + + if not failed: + retstatus = 0 + else: + retstatus = 1 + return retstatus + + except Exception, e: + logging.info("Traceback " + traceback.format_exc()) + return 1 + pass + + def changePasswordFileModeOnHost(self): + try: + # Change password file mode to 600 + logging.info("Changing password file mode...") + targetDir = self.getRepoDir() + command = "chmod 600 " + self.getPasswordFile() + pssh = PSSH(self.successive_hostlist, self.user, self.sshkeyFile, command, self.bootdir) + pssh.run() + out = pssh.getstatus() + # Prepearing report about failed hosts + failed_current = get_difference(self.successive_hostlist, skip_failed_hosts(out)) + self.successive_hostlist = skip_failed_hosts(out) + failed = get_difference(self.hostlist, self.successive_hostlist) + logging.warning("Parallel scp returns for copying password file. All failed hosts are: " + str(failed) + + ". Failed on last step: " + str(failed_current)) + #updating statuses + self.statuses = unite_statuses(self.statuses, out) + + if not failed: + retstatus = 0 + else: + retstatus = 1 + return retstatus + + except Exception, e: + logging.info("Traceback " + traceback.format_exc()) + return 1 + pass + + def deletePasswordFile(self): + try: + # Deleting the password file + logging.info("Deleting password file...") + targetDir = self.getRepoDir() + command = "rm " + self.getPasswordFile() + pssh = PSSH(self.hostlist_to_remove_password_file, self.user, self.sshkeyFile, command, self.bootdir) + pssh.run() + out = pssh.getstatus() + # Prepearing report about failed hosts + failed_current = get_difference(self.hostlist_to_remove_password_file, skip_failed_hosts(out)) + self.successive_hostlist = skip_failed_hosts(out) + failed = get_difference(self.hostlist, self.successive_hostlist) + logging.warn("Parallel scp returns for deleting password file. All failed hosts are: " + str(failed) + + ". Failed on last step: " + str(failed_current)) + #updating statuses + self.statuses = unite_statuses(self.statuses, out) + + if not failed: + retstatus = 0 + else: + retstatus = 1 + return retstatus + + except Exception, e: + logging.info("Traceback " + traceback.format_exc()) + return 1 + pass + def run(self): """ Copy files and run commands on remote hosts """ - ret1 = self.copyNeededFiles() + ret1 = self.checkSudoPackage() + logging.info("Checking 'sudo' package finished") + ret2 = 0 + ret3 = 0 + if self.hasPassword(): + ret2 = self.copyPasswordFile() + logging.info("Copying password file finished") + ret3 = self.changePasswordFileModeOnHost() + logging.info("Change password file mode on host finished") + ret4 = self.copyNeededFiles() logging.info("Copying files finished") - ret2 = self.runSetupAgent() + ret5 = self.runSetupAgent() logging.info("Running ssh command finished") - retcode = max(ret1, ret2) + ret6 = 0 + if self.hasPassword() and self.hostlist_to_remove_password_file is not None: + ret6 = self.deletePasswordFile() + logging.info("Deleting password file finished") + retcode = max(ret1, ret2, ret3, ret4, ret5, ret6) self.createDoneFiles() return retcode pass @@ -352,23 +564,29 @@ def main(argv=None): onlyargs = argv[1:] if len(onlyargs) < 3: sys.stderr.write("Usage: " - " \n") + " \n") sys.exit(2) pass #Parse the input hostList = onlyargs[0].split(",") bootdir = onlyargs[1] - sshKeyFile = onlyargs[2] - setupAgentFile = onlyargs[3] - ambariServer = onlyargs[4] + user = onlyargs[2] + sshKeyFile = onlyargs[3] + setupAgentFile = onlyargs[4] + ambariServer = onlyargs[5] + passwordFile = onlyargs[6] # ssh doesn't like open files stat = subprocess.Popen(["chmod", "600", sshKeyFile], stdout=subprocess.PIPE) + + if passwordFile != None and passwordFile != 'null': + stat = subprocess.Popen(["chmod", "600", passwordFile], stdout=subprocess.PIPE) logging.info("BootStrapping hosts " + pprint.pformat(hostList) + "using " + scriptDir + - " with sshKey File " + sshKeyFile + " using tmp dir " + bootdir + " ambari: " + ambariServer) - bootstrap = BootStrap(hostList, sshKeyFile, scriptDir, bootdir, setupAgentFile, ambariServer) + " with user '" + user + "' sshKey File " + sshKeyFile + " password File " + passwordFile + + " using tmp dir " + bootdir + " ambari: " + ambariServer) + bootstrap = BootStrap(hostList, user, sshKeyFile, scriptDir, bootdir, setupAgentFile, ambariServer, passwordFile) ret = bootstrap.run() #return ret return 0 # Hack to comply with current usage Modified: incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/bootstrap/BootStrapTest.java URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/bootstrap/BootStrapTest.java?rev=1467531&r1=1467530&r2=1467531&view=diff ============================================================================== --- incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/bootstrap/BootStrapTest.java (original) +++ incubator/ambari/trunk/ambari-server/src/test/java/org/apache/ambari/server/bootstrap/BootStrapTest.java Sat Apr 13 00:46:21 2013 @@ -73,6 +73,8 @@ public class BootStrapTest extends TestC hosts.add("host1"); hosts.add("host2"); info.setHosts(hosts); + info.setUser("user"); + info.setPassword("passwd"); BSResponse response = impl.runBootStrap(info); LOG.info("Response id from bootstrap " + response.getRequestId()); /* do a query */ @@ -90,6 +92,7 @@ public class BootStrapTest extends TestC Assert.assertTrue(status.getLog().contains("host1,host2")); Assert.assertEquals(BSStat.SUCCESS, status.getStatus()); Assert.assertFalse(new File(bootdir + File.separator + "1" + File.separator + "sshKey").exists()); + Assert.assertFalse(new File(bootdir + File.separator + "1" + File.separator + "host_pass").exists()); } Modified: incubator/ambari/trunk/ambari-server/src/test/python/TestBootstrap.py URL: http://svn.apache.org/viewvc/incubator/ambari/trunk/ambari-server/src/test/python/TestBootstrap.py?rev=1467531&r1=1467530&r2=1467531&view=diff ============================================================================== --- incubator/ambari/trunk/ambari-server/src/test/python/TestBootstrap.py (original) +++ incubator/ambari/trunk/ambari-server/src/test/python/TestBootstrap.py Sat Apr 13 00:46:21 2013 @@ -18,21 +18,31 @@ limitations under the License. import bootstrap import time +import subprocess +import os +import logging from bootstrap import SCP from bootstrap import PSCP from bootstrap import SSH from bootstrap import PSSH +from bootstrap import BootStrap from unittest import TestCase +from subprocess import Popen +from bootstrap import AMBARI_PASSPHRASE_VAR_NAME +from mock.mock import MagicMock, call class TestBootstrap(TestCase): + def setUp(self): + logging.basicConfig(level=logging.ERROR) + #Timout is specified in bootstrap.HOST_BOOTSTRAP_TIMEOUT, default is 300 seconds def test_return_failed_status_for_hanging_ssh_threads_after_timeout(self): bootstrap.HOST_BOOTSTRAP_TIMEOUT = 1 forever_hanging_timeout = 5 SSH.run = lambda self: time.sleep(forever_hanging_timeout) - pssh = PSSH(["hostname"], "sshKeyFile", "command", "bootdir") + pssh = PSSH(["hostname"], "root", "sshKeyFile", "command", "bootdir") self.assertTrue(pssh.ret == {}) starttime = time.time() pssh.run() @@ -46,7 +56,7 @@ class TestBootstrap(TestCase): bootstrap.HOST_BOOTSTRAP_TIMEOUT = 1 forever_hanging_timeout = 5 SCP.run = lambda self: time.sleep(forever_hanging_timeout) - pscp = PSCP(["hostname"], "sshKeyFile", "inputfile", "remote", "bootdir") + pscp = PSCP(["hostname"], "root", "sshKeyFile", "inputfile", "remote", "bootdir") self.assertTrue(pscp.ret == {}) starttime = time.time() pscp.run() @@ -54,3 +64,114 @@ class TestBootstrap(TestCase): self.assertTrue(time.time() - starttime < forever_hanging_timeout) self.assertTrue(pscp.ret["hostname"]["log"] == "FAILED") self.assertTrue(pscp.ret["hostname"]["exitstatus"] == -1) + + def test_return_error_message_for_missing_sudo_package(self): + Popen.communicate = lambda self: ("", "") + SCP.writeLogToFile = lambda self, logFilePath: None + SSH.writeLogToFile = lambda self, logFilePath: None + SSH.writeDoneToFile = lambda self, doneFilePath, returncode: None + bootstrap = BootStrap(["hostname"], "root", "sshKeyFile", "scriptDir", "bootdir", "setupAgentFile", "ambariServer") + ret = bootstrap.checkSudoPackage() + self.assertTrue("Error: Sudo command is not available. Please install the sudo command." in bootstrap.statuses["hostname"]["log"]) + + def test_copy_and_delete_password_file_methods_are_called_for_user_with_password(self): + Popen.communicate = lambda self: ("", "") + SCP.writeLogToFile = lambda self, logFilePath: None + SSH.writeLogToFile = lambda self, logFilePath: None + SSH.writeDoneToFile = lambda self, doneFilePath, returncode: None + BootStrap.createDoneFiles = lambda self: None + + def side_effect(self): + self.copyPasswordFile_called = True + self.hostlist_to_remove_password_file = ["hostname"] + return 0 + BootStrap.copyPasswordFile = side_effect + + deletePasswordFile = MagicMock() + deletePasswordFile.return_value = 0 + BootStrap.deletePasswordFile = deletePasswordFile + + changePasswordFileModeOnHost = MagicMock() + changePasswordFileModeOnHost.return_value = 0 + BootStrap.changePasswordFileModeOnHost = changePasswordFileModeOnHost + + os.environ[AMBARI_PASSPHRASE_VAR_NAME] = "" + bootstrap = BootStrap(["hostname"], "user", "sshKeyFile", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "passwordFile") + ret = bootstrap.run() + self.assertTrue(bootstrap.copyPasswordFile_called) + self.assertTrue(deletePasswordFile.called) + self.assertTrue(changePasswordFileModeOnHost.called) + + def test_copy_and_delete_password_file_methods_are_not_called_for_passwordless_user(self): + Popen.communicate = lambda self: ("", "") + SCP.writeLogToFile = lambda self, logFilePath: None + SSH.writeLogToFile = lambda self, logFilePath: None + SSH.writeDoneToFile = lambda self, doneFilePath, returncode: None + BootStrap.createDoneFiles = lambda self: None + + def side_effect(self): + self.copyPasswordFile_called = True + self.hostlist_to_remove_password_file = ["hostname"] + return 0 + BootStrap.copyPasswordFile = side_effect + + deletePasswordFile = MagicMock() + deletePasswordFile.return_value = 0 + BootStrap.deletePasswordFile = deletePasswordFile + + changePasswordFileModeOnHost = MagicMock() + changePasswordFileModeOnHost.return_value = 0 + BootStrap.changePasswordFileModeOnHost = changePasswordFileModeOnHost + + os.environ[AMBARI_PASSPHRASE_VAR_NAME] = "" + bootstrap = BootStrap(["hostname"], "user", "sshKeyFile", "scriptDir", "bootdir", "setupAgentFile", "ambariServer") + bootstrap.copyPasswordFile_called = False + ret = bootstrap.run() + self.assertFalse(bootstrap.copyPasswordFile_called) + self.assertFalse(deletePasswordFile.called) + self.assertFalse(changePasswordFileModeOnHost.called) + + def test_commands_with_password_are_called_for_user_with_password(self): + def communicate(self, input=None, timeout=None): + self.returncode = 0 + return ("", "") + Popen.communicate = communicate + SCP.writeLogToFile = lambda self, logFilePath: None + SSH.writeLogToFile = lambda self, logFilePath: None + SSH.writeDoneToFile = lambda self, doneFilePath, returncode: None + BootStrap.createDoneFiles = lambda self: None + + getRunSetupWithPasswordCommand = MagicMock() + getRunSetupWithPasswordCommand.return_value = "" + BootStrap.getRunSetupWithPasswordCommand = getRunSetupWithPasswordCommand + + getMoveRepoFileWithPasswordCommand = MagicMock() + getMoveRepoFileWithPasswordCommand.return_value = "" + BootStrap.getMoveRepoFileWithPasswordCommand = getMoveRepoFileWithPasswordCommand + + os.environ[AMBARI_PASSPHRASE_VAR_NAME] = "" + bootstrap = BootStrap(["hostname"], "user", "sshKeyFile", "scriptDir", "bootdir", "setupAgentFile", "ambariServer", "passwordFile") + ret = bootstrap.run() + self.assertTrue(getRunSetupWithPasswordCommand.called) + self.assertTrue(getMoveRepoFileWithPasswordCommand.called) + + def test_commands_without_password_are_called_for_passwordless_user(self): + Popen.communicate = lambda self: ("", "") + SCP.writeLogToFile = lambda self, logFilePath: None + SSH.writeLogToFile = lambda self, logFilePath: None + SSH.writeDoneToFile = lambda self, doneFilePath, returncode: None + BootStrap.createDoneFiles = lambda self: None + + getRunSetupWithoutPasswordCommand = MagicMock() + getRunSetupWithoutPasswordCommand.return_value = "" + BootStrap.getRunSetupWithoutPasswordCommand = getRunSetupWithoutPasswordCommand + + getMoveRepoFileWithoutPasswordCommand = MagicMock() + getMoveRepoFileWithoutPasswordCommand.return_value = "" + BootStrap.getMoveRepoFileWithoutPasswordCommand = getMoveRepoFileWithoutPasswordCommand + + os.environ[AMBARI_PASSPHRASE_VAR_NAME] = "" + bootstrap = BootStrap(["hostname"], "user", "sshKeyFile", "scriptDir", "bootdir", "setupAgentFile", "ambariServer") + ret = bootstrap.run() + self.assertTrue(getRunSetupWithoutPasswordCommand.called) + self.assertTrue(getMoveRepoFileWithoutPasswordCommand.called)