Return-Path: X-Original-To: apmail-incubator-ambari-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-ambari-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 3757EDBF8 for ; Wed, 7 Nov 2012 08:28:06 +0000 (UTC) Received: (qmail 11571 invoked by uid 500); 7 Nov 2012 08:28:06 -0000 Delivered-To: apmail-incubator-ambari-commits-archive@incubator.apache.org Received: (qmail 11525 invoked by uid 500); 7 Nov 2012 08:28:06 -0000 Mailing-List: contact ambari-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: ambari-dev@incubator.apache.org Delivered-To: mailing list ambari-commits@incubator.apache.org Received: (qmail 11516 invoked by uid 99); 7 Nov 2012 08:28:05 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 07 Nov 2012 08:28:05 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Wed, 07 Nov 2012 08:28:01 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id A8F7F23888CD; Wed, 7 Nov 2012 08:27:41 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit Subject: svn commit: r1406492 [1/2] - in /incubator/ambari/branches/AMBARI-666: ./ ambari-agent/ ambari-agent/conf/unix/ ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/ ambari-agent/src/main/puppet/modules/hdp-repos/templates/ ambari-agent/src/main/pu... Date: Wed, 07 Nov 2012 08:27:38 -0000 To: ambari-commits@incubator.apache.org From: mahadev@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20121107082741.A8F7F23888CD@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: mahadev Date: Wed Nov 7 08:27:36 2012 New Revision: 1406492 URL: http://svn.apache.org/viewvc?rev=1406492&view=rev Log: AMBARI-971. Add api support for creating multiple resources in a single request. (John Speidel via mahadev) Added: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/RepoInstaller.py Modified: incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt incubator/ambari/branches/AMBARI-666/ambari-agent/conf/unix/ambari-agent incubator/ambari/branches/AMBARI-666/ambari-agent/conf/unix/ambari.ini incubator/ambari/branches/AMBARI-666/ambari-agent/pom.xml incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-repos/templates/repo.erb incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py incubator/ambari/branches/AMBARI-666/ambari-server/conf/unix/ambari.properties incubator/ambari/branches/AMBARI-666/ambari-server/pom.xml incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/BaseManagementHandler.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ComponentResourceDefinition.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ConfigurationResourceDefinition.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostComponentResourceDefinition.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostResourceDefinition.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceDefinition.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/BasePersistenceManager.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/ConfigurationService.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/CreatePersistenceManager.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/DeletePersistenceManager.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostComponentService.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostService.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/PersistKeyValueService.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/PersistenceManager.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/Request.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/RequestImpl.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/UpdatePersistenceManager.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/JsonPropertyParser.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/RequestBodyParser.java incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/ambari-server.py incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/bootstrap.py incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/setupAgent.py incubator/ambari/branches/AMBARI-666/ambari-server/src/main/resources/ca.config incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/api/handlers/CreateHandlerTest.java incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/api/handlers/DeleteHandlerTest.java incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/api/handlers/UpdateHandlerTest.java incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/api/query/QueryImplTest.java incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/api/services/ConfigurationServiceTest.java incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/api/services/CreatePersistenceManagerTest.java incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/api/services/DeletePersistenceManagerTest.java incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/api/services/HostServiceTest.java incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/api/services/UpdatePersistenceManagerTest.java incubator/ambari/branches/AMBARI-666/ambari-server/src/test/java/org/apache/ambari/server/api/services/parsers/JsonPropertyParserTest.java Modified: incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt (original) +++ incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt Wed Nov 7 08:27:36 2012 @@ -12,6 +12,9 @@ AMBARI-666 branch (unreleased changes) NEW FEATURES + AMBARI-971. Add api support for creating multiple resources in a single + request. (John Speidel via mahadev) + AMBARI-970. Add additional Ganglia metrics and JMX properties. (Tom Beerbower via mahadev) Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/conf/unix/ambari-agent URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/conf/unix/ambari-agent?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-agent/conf/unix/ambari-agent (original) +++ incubator/ambari/branches/AMBARI-666/ambari-agent/conf/unix/ambari-agent Wed Nov 7 08:27:36 2012 @@ -6,7 +6,6 @@ case "$1" in start) echo -e "Starting ambari-agent" - export AMBARI_PASSPHRASE=pass_phrase python /usr/lib/python2.6/site-packages/ambari_agent/main.py ;; stop) Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/conf/unix/ambari.ini URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/conf/unix/ambari.ini?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-agent/conf/unix/ambari.ini (original) +++ incubator/ambari/branches/AMBARI-666/ambari-agent/conf/unix/ambari.ini Wed Nov 7 08:27:36 2012 @@ -10,7 +10,7 @@ prefix=/tmp/ambari-agent installprefix=/var/ambari/ [puppet] -puppetmodules=/etc/ambari-agent/puppet +puppetmodules=/var/lib/ambari-agent/puppet puppet_home=/usr/bin/puppet facter_home=/usr/bin/facter @@ -19,7 +19,7 @@ maxretries=2 sleepBetweenRetries=1 [security] -keysdir=/etc/ambari-agent/keys +keysdir=/var/lib/ambari-agent/keys server_crt=ca.crt passphrase_env_var_name=AMBARI_PASSPHRASE Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/pom.xml URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/pom.xml?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-agent/pom.xml (original) +++ incubator/ambari/branches/AMBARI-666/ambari-agent/pom.xml Wed Nov 7 08:27:36 2012 @@ -113,6 +113,9 @@ 2012, Apache Software Foundation Development Maven Recipe: RPM Package. + + puppet = 2.7.9 + @@ -125,7 +128,7 @@ - /etc/${project.artifactId}/puppet + /var/lib/${project.artifactId}/puppet src/main/puppet @@ -155,9 +158,14 @@ /var/run/ambari - - /etc/ambari-agent/keys + /var/lib/${project.artifactId}/keys + + + /var/log/ambari + + + /var/ambari Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp (original) +++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-hbase/manifests/init.pp Wed Nov 7 08:27:36 2012 @@ -40,9 +40,6 @@ class hdp-hbase( configuration => $configuration['hbase-site'] } } - hdp-hbase::configfile { 'regionservers':} - Anchor['hdp-hbase::begin'] -> Hdp::Package['hbase'] -> Hdp::User[$hbase_user] -> Hdp::Directory[$config_dir] -> - Hdp-hbase::Configfile<||> -> Anchor['hdp-hbase::end'] if has_key($configuration, 'hbase-policy') { configgenerator::configfile{'hbase-policy': @@ -91,7 +88,7 @@ define hdp-hbase::configfile( $mode = undef, $hbase_master_host = undef, $template_tag = undef, - $type = undef, + $type = undef ) { if ($name == 'hadoop-metrics.properties') { Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-repos/templates/repo.erb URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-repos/templates/repo.erb?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-repos/templates/repo.erb (original) +++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp-repos/templates/repo.erb Wed Nov 7 08:27:36 2012 @@ -22,6 +22,5 @@ [<%=repo_id%>] name=<%=repo_name %> baseurl=<%=base_url %> +path=/ enabled=1 - - Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp (original) +++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/puppet/modules/hdp/manifests/params.pp Wed Nov 7 08:27:36 2012 @@ -67,6 +67,9 @@ class hdp::params() /^6\..+$/: { $hdp_os_type = "rhel6" } } } + suse: { + $hdp_os_type = "suse" + } default: { hdp_fail("No support for os ${hdp_os} ${hdp_os_version}") } @@ -354,7 +357,8 @@ class hdp::params() $repos_paths = { - centos6 => '/etc/yum.repos.d' + centos6 => '/etc/yum.repos.d', + suse => '/etc/zypp/repos.d' } } Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py (original) +++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/ActionQueue.py Wed Nov 7 08:27:36 2012 @@ -133,6 +133,10 @@ class ActionQueue(threading.Thread): 'exitCode' : commandresult['exitcode'], 'serviceName' : serviceName, 'status' : status} + if roleResult['stdout'] == '': + roleResult['stdout'] = 'None' + if roleResult['stderr'] == '': + roleResult['stderr'] = 'None' result.append(roleResult) pass return result Added: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/RepoInstaller.py URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/RepoInstaller.py?rev=1406492&view=auto ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/RepoInstaller.py (added) +++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/RepoInstaller.py Wed Nov 7 08:27:36 2012 @@ -0,0 +1,79 @@ +#!/usr/bin/env python2.6 + +''' +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +''' + +import logging +import os +import json +from shell import shellRunner +from manifestGenerator import writeImports + + +PUPPET_EXT=".pp" + +logger = logging.getLogger() + +class RepoInstaller: + def __init__(self, parsedJson, path, modulesdir, taskId): + self.parsedJson = parsedJson + self.path = path + self.modulesdir = modulesdir + self.taskId = taskId + self.sh = shellRunner() + + def prepareReposInfo(self): + params = {} + self.repoInfoList = [] + if self.parsedJson.has_key('hostLevelParams'): + params = self.parsedJson['hostLevelParams'] + if params.has_key('repo_info'): + self.repoInfoList = params['repo_info'] + + def generateFiles(self): + repoPuppetFiles = [] + for repo in self.repoInfoList: + repoFile = open(self.path + os.sep + repo['repo_id'] + '-' + str(self.taskId) + PUPPET_EXT, 'w+') + writeImports(repoFile, self.modulesdir, inputFileName='imports.txt') + repoFile.write('node /default/ {') + repoFile.write('class{ "hdp-repos::process_repo" : ' + ' os_type => "' + repo['os_type'] + + '", repo_id => "' + repo['repo_id'] + '", base_url => "' + repo['base_url'] + + '", repo_name => "' + repo['repo_name'] + '" }' ) + repoFile.write('}') + repoFile.close() + repoPuppetFiles.append(repoFile.name) + + return repoPuppetFiles + + def installRepos(self): + self.prepareReposInfo() + repoPuppetFiles = self.generateFiles() + return repoPuppetFiles + +def main(): + #Test code + jsonFile = open('test.json', 'r') + jsonStr = jsonFile.read() + parsedJson = json.loads(jsonStr) + repoInstaller = RepoInstaller(parsedJson, '/tmp', '/home/centos/ambari_repo_info/ambari-agent/src/main/puppet/modules') + repoInstaller.installRepos() + +if __name__ == '__main__': + main() + + Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py (original) +++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py Wed Nov 7 08:27:36 2012 @@ -24,8 +24,6 @@ import logging from uuid import getnode as get_mac from shell import shellRunner -REPO_INFO_DIR="repos_info" -PUPPET_EXT=".pp" logger = logging.getLogger() @@ -138,9 +136,7 @@ def writeNodes(outputFile, clusterHostIn def writeParams(outputFile, params, modulesdir): for paramName in params.iterkeys(): - # todo handle repo information properly - if paramName == 'repo_info': - processRepo(params[paramName],modulesdir) + if paramName == 'repo_info': continue @@ -257,39 +253,7 @@ def writeStages(outputFile, numStages): outputFile.write('\n') -def processRepo(repoInfoList, modulesdir): - if not os.path.exists(REPO_INFO_DIR): - os.makedirs(REPO_INFO_DIR) - - for repo in repoInfoList: - - repoFile = open(REPO_INFO_DIR + os.sep + repo['repo_id'] + PUPPET_EXT, 'w+') - writeImports(repoFile, modulesdir, inputFileName='imports.txt') - repoFile.write('node /default/ {') - repoFile.write('class{ "hdp-repos::process_repo" : ' + ' os_type => "' + repo['os_type'] + - '", repo_id => "' + repo['repo_id'] + '", base_url => "' + repo['base_url'] + - '", repo_name => "' + repo['repo_name'] + '" }' ) - repoFile.write('}') - repoFile.close() - - -def installRepos(): - sh = shellRunner() - agentdir = os.getcwd() - confdir = os.path.abspath(agentdir + ".." + os.sep + ".." + - os.sep + ".." + os.sep + "puppet") - - for repo in os.listdir(REPO_INFO_DIR): - if not repo.endswith(PUPPET_EXT): - continue - logfile = repo + '_log.log' - res = sh.run(['puppet apply', '--confdir=' + confdir, REPO_INFO_DIR + os.sep + repo, - '--logdest=' + agentdir + os.sep + REPO_INFO_DIR + os.sep + logfile]) - if res['exitCode'] == 0: - logger.info('Repository ' + repo + ' was installed') - else: - logger.error('Repository ' + repo + ' wasn''t installed. Please find detailed info in logfile:' + logfile) def main(): logging.basicConfig(level=logging.DEBUG) Modified: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py (original) +++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/puppetExecutor.py Wed Nov 7 08:27:36 2012 @@ -22,6 +22,7 @@ import os.path import logging import subprocess from manifestGenerator import generateManifest +from RepoInstaller import RepoInstaller import pprint from Grep import Grep @@ -29,6 +30,9 @@ logger = logging.getLogger() class puppetExecutor: + """ Class that executes the commands that come from the server using puppet. + This is the class that provides the pluggable point for executing the puppet""" + # How many lines from command output send to server OUTPUT_LAST_LINES = 10 # How many lines from command error output send to server (before Err phrase) @@ -38,9 +42,6 @@ class puppetExecutor: NO_ERROR = "none" - """ Class that executes the commands that come from the server using puppet. - This is the class that provides the pluggable point for executing the puppet""" - def __init__(self, puppetModule, puppetInstall, facterInstall, tmpDir): self.puppetModule = puppetModule self.puppetInstall = puppetInstall @@ -76,39 +77,70 @@ class puppetExecutor: taskId = command['taskId'] puppetEnv = os.environ + #Install repos + modulesdir = self.puppetModule + "/modules" + repoInstaller = RepoInstaller(command, self.tmpDir, modulesdir, taskId) + + puppetFiles = repoInstaller.installRepos() siteppFileName = os.path.join(self.tmpDir, "site-" + str(taskId) + ".pp") - generateManifest(command, siteppFileName, self.puppetModule + "/modules") - puppetcommand = self.puppetCommand(siteppFileName) - """ Run the command and make sure the output gets propagated""" - rubyLib = "" - if os.environ.has_key("RUBYLIB"): - rubyLib = os.environ["RUBYLIB"] - logger.info("Ruby Lib env from Env " + rubyLib) - rubyLib = rubyLib + ":" + self.facterLib() + ":" + self.puppetLib() - puppetEnv["RUBYLIB"] = rubyLib - logger.info("Setting RUBYLIB as: " + rubyLib) - logger.info("Running command " + pprint.pformat(puppetcommand)) - puppet = subprocess.Popen(puppetcommand, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - env=puppetEnv) - stderr_out = puppet.communicate() - error = self.NO_ERROR - returncode = 0 - if puppet.returncode != 0 and puppet.returncode != 2: - returncode = puppet.returncode - error = stderr_out[1] - logging.error("Error running puppet: \n" + stderr_out[1]) - pass - result["stderr"] = error - puppetOutput = stderr_out[0] - logger.info("Output from puppet :\n" + puppetOutput) - result["exitcode"] = returncode + puppetFiles.append(siteppFileName) + generateManifest(command, siteppFileName, modulesdir) + #Run all puppet commands, from manifest generator and for repos installation + #Appending outputs and errors, exitcode - maximal from all + for puppetFile in puppetFiles: + puppetcommand = self.puppetCommand(puppetFile) + """ Run the command and make sure the output gets propagated""" + rubyLib = "" + if os.environ.has_key("RUBYLIB"): + rubyLib = os.environ["RUBYLIB"] + logger.info("Ruby Lib env from Env " + rubyLib) + rubyLib = rubyLib + ":" + self.facterLib() + ":" + self.puppetLib() + puppetEnv["RUBYLIB"] = rubyLib + logger.info("Setting RUBYLIB as: " + rubyLib) + logger.info("Running command " + pprint.pformat(puppetcommand)) + puppet = subprocess.Popen(puppetcommand, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=puppetEnv) + stderr_out = puppet.communicate() + error = "none" + returncode = 0 + if (puppet.returncode != 0 and puppet.returncode != 2) : + returncode = puppet.returncode + error = stderr_out[1] + logging.error("Error running puppet: \n" + stderr_out[1]) + pass + + if result.has_key("stderr"): + result["stderr"] = result["stderr"] + os.linesep + error + else: + result["stderr"] = error + puppetOutput = stderr_out[0] + logger.info("Output from puppet :\n" + puppetOutput) + if result.has_key("exitcode"): + result["exitcode"] = max(returncode, result["exitcode"]) + else: + result["exitcode"] = returncode + + + if result.has_key("stdout"): + result["stdout"] = result["stdout"] + os.linesep + puppetOutput + else: + result["stdout"] = puppetOutput + if error == self.NO_ERROR: - result["stdout"] = grep.tail(puppetOutput, self.OUTPUT_LAST_LINES) + if result.has_key("stdout"): + result["stdout"] = result["stdout"] + os.linesep + str(grep.tail(puppetOutput, self.OUTPUT_LAST_LINES)) + else: + result["stdout"] = grep.tail(puppetOutput, self.OUTPUT_LAST_LINES) else: - result["stdout"] = grep.grep(puppetOutput, "err", self.ERROR_LAST_LINES_BEFORE, self.ERROR_LAST_LINES_AFTER) + if result.has_key("stdout"): + result["stdout"] = result["stdout"] + os.linesep + str(grep.grep(puppetOutput, "err", self.ERROR_LAST_LINES_BEFORE, self.ERROR_LAST_LINES_AFTER)) + else: + result["stdout"] = str(grep.grep(puppetOutput, "err", self.ERROR_LAST_LINES_BEFORE, self.ERROR_LAST_LINES_AFTER)) + logger.info("ExitCode : " + str(result["exitcode"])) + return result def main(): @@ -118,14 +150,15 @@ def main(): jsonStr = jsonFile.read() # Below is for testing only. - puppetInstance = puppetExecutor("/root/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/puppet/", - "/root/workspace/puppet-install/puppet-2.7.9", + puppetInstance = puppetExecutor("/home/centos/ambari_repo_info/ambari-agent/src/main/puppet/", + "/usr/", "/root/workspace/puppet-install/facter-1.6.10/", "/tmp") jsonFile = open('test.json', 'r') jsonStr = jsonFile.read() parsedJson = json.loads(jsonStr) - puppetInstance.runCommand(parsedJson) + result = puppetInstance.runCommand(parsedJson) + logger.debug(result) if __name__ == '__main__': main() Modified: incubator/ambari/branches/AMBARI-666/ambari-server/conf/unix/ambari.properties URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/conf/unix/ambari.properties?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/conf/unix/ambari.properties (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/conf/unix/ambari.properties Wed Nov 7 08:27:36 2012 @@ -1,3 +1,4 @@ security.server.keys_dir = /var/lib/ambari-server/keys resources.dir = /var/lib/ambari-server/resources jdk.url=http://public-repo-1.hortonworks.com/ARTIFACTS/jdk-6u31-linux-x64.bin +metadata.path=/var/lib/ambari-server/resources/stacks Modified: incubator/ambari/branches/AMBARI-666/ambari-server/pom.xml URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/pom.xml?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/pom.xml (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/pom.xml Wed Nov 7 08:27:36 2012 @@ -22,7 +22,7 @@ ambari-server jar Ambari Server - 1.0.3-SNAPSHOT + 1.0.3 Ambari Server @@ -75,6 +75,7 @@ --> 2012, Apache Software Foundation + ${project.version} Development Maven Recipe: RPM Package. @@ -155,9 +156,25 @@ src/main/resources/Ambari-DDL.sql + + src/main/resources/stacks + + + + + + /usr/lib/python2.6/site-packages/ambari_server + + + src/main/python/bootstrap.py + + + src/main/python/setupAgent.py + + /var/run/ambari-server Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/BaseManagementHandler.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/BaseManagementHandler.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/BaseManagementHandler.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/handlers/BaseManagementHandler.java Wed Nov 7 08:27:36 2012 @@ -23,10 +23,13 @@ import org.apache.ambari.server.api.serv import org.apache.ambari.server.api.services.Result; import org.apache.ambari.server.api.services.ResultImpl; import org.apache.ambari.server.api.util.TreeNode; +import org.apache.ambari.server.controller.spi.PropertyId; import org.apache.ambari.server.controller.spi.RequestStatus; import org.apache.ambari.server.controller.spi.Resource; import org.apache.ambari.server.controller.utilities.PropertyHelper; +import java.util.Collections; +import java.util.Map; import java.util.Set; /** @@ -36,10 +39,8 @@ public class BaseManagementHandler imple @Override public Result handleRequest(Request request) { ResourceDefinition resource = request.getResourceDefinition(); - resource.setProperties(request.getHttpBodyProperties()); - RequestStatus status = request.getPersistenceManager().persist(resource); - - return createResult(request, status); + return createResult(request, request.getPersistenceManager().persist( + resource, request.getHttpBodyProperties())); } private Result createResult(Request request, RequestStatus requestStatus) { @@ -62,14 +63,12 @@ public class BaseManagementHandler imple if (! isSynchronous) { Resource requestResource = requestStatus.getRequestResource(); TreeNode r = tree.addChild(requestResource, "request"); - String requestHref = buildRequestHref(request, requestStatus); - r.setProperty("href", requestHref); + r.setProperty("href", buildRequestHref(request, requestStatus)); } return result; } - //todo: this needs to be rewritten and needs to support operating on clusters collection private String buildRequestHref(Request request, RequestStatus requestStatus) { StringBuilder sb = new StringBuilder(); String origHref = request.getURI(); Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/query/QueryImpl.java Wed Nov 7 08:27:36 2012 @@ -21,7 +21,6 @@ package org.apache.ambari.server.api.que import org.apache.ambari.server.api.resources.ResourceDefinition; import org.apache.ambari.server.api.services.ResultImpl; import org.apache.ambari.server.api.util.TreeNodeImpl; -import org.apache.ambari.server.controller.internal.ClusterControllerImpl; import org.apache.ambari.server.controller.internal.PropertyIdImpl; import org.apache.ambari.server.controller.utilities.ClusterControllerHelper; import org.apache.ambari.server.controller.utilities.PropertyHelper; @@ -32,7 +31,6 @@ import org.apache.ambari.server.controll import org.apache.ambari.server.api.services.Result; import org.apache.ambari.server.controller.spi.*; import org.apache.ambari.server.api.util.TreeNode; -import org.mortbay.log.Log; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -155,21 +153,12 @@ public class QueryImpl implements Query // add a child node for the resource and provide a unique name. The name is never used. //todo: provide a more meaningful node name TreeNode node = tree.addChild(resource, resource.getType() + ":" + count++); - LOG.info("Resource object resource " + resource); for (Map.Entry entry : m_mapSubResources.entrySet()) { String subResCategory = entry.getKey(); ResourceDefinition r = entry.getValue(); - - r.setParentId(m_resourceDefinition.getType(), (String) (resource.getPropertyValue( - getClusterController().getSchema(m_resourceDefinition.getType()). - getKeyPropertyId(m_resourceDefinition.getType())).toString())); - - LOG.info("Setting various values for resource " + r.getId()); - if (r.getResourceIds() != null) { - for (Map.Entry tentry: r.getResourceIds().entrySet()) { - LOG.info("Resource Id's " + tentry.getKey() + " value " + tentry.getValue()); - } - } + + setParentIdsOnSubResource(resource, r); + TreeNode childResult = r.getQuery().execute().getResultTree(); childResult.setName(subResCategory); childResult.setProperty("isCollection", "false"); @@ -178,7 +167,6 @@ public class QueryImpl implements Query } return result; } - @Override public Predicate getInternalPredicate() { return createInternalPredicate(m_resourceDefinition); @@ -272,7 +260,6 @@ public class QueryImpl implements Query } private BasePredicate createInternalPredicate(ResourceDefinition resourceDefinition) { - //todo: account for user predicates Resource.Type resourceType = resourceDefinition.getType(); Map mapResourceIds = resourceDefinition.getResourceIds(); Schema schema = getClusterController().getSchema(resourceType); @@ -283,7 +270,10 @@ public class QueryImpl implements Query //todo: host_component queries and host is not available for component queries. //todo: this should be rectified when the data model is changed for host_component if (entry.getValue() != null) { - setPredicates.add(new EqualsPredicate(schema.getKeyPropertyId(entry.getKey()), entry.getValue())); + PropertyId keyPropertyId = schema.getKeyPropertyId(entry.getKey()); + if (keyPropertyId != null) { + setPredicates.add(new EqualsPredicate(keyPropertyId, entry.getValue())); + } } } @@ -358,6 +348,24 @@ public class QueryImpl implements Query return r; } + private void setParentIdsOnSubResource(Resource resource, ResourceDefinition r) { + Map mapParentIds = m_resourceDefinition.getResourceIds(); + Map mapResourceIds = new HashMap(mapParentIds.size()); + for (Map.Entry resourceIdEntry : mapParentIds.entrySet()) { + Resource.Type type = resourceIdEntry.getKey(); + String value = resourceIdEntry.getValue(); + + if (value == null) { + Object o = resource.getPropertyValue(getClusterController().getSchema(type).getKeyPropertyId(type)); + value = o == null ? null : o.toString(); + } + if (value != null) { + mapResourceIds.put(type, value); + } + } + r.setParentIds(mapResourceIds); + } + Result createResult() { return new ResultImpl(true); } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/BaseResourceDefinition.java Wed Nov 7 08:27:36 2012 @@ -56,10 +56,6 @@ public abstract class BaseResourceDefini */ private Map m_mapResourceIds = new HashMap(); - //TODO: Refactor out of this class when setProperties is moved. - private Map m_properties = new HashMap(); - - /** * Constructor. * @@ -73,8 +69,10 @@ public abstract class BaseResourceDefini } @Override - public void setParentId(Resource.Type type, String value) { - setResourceId(type, value); + public void setParentIds(Map mapIds) { + for (Map.Entry entry : mapIds.entrySet() ) { + setResourceId(entry.getKey(), entry.getValue()); + } } @Override @@ -119,22 +117,6 @@ public abstract class BaseResourceDefini return listProcessors; } - //todo: refactor set/get property methods out of this class - @Override - public void setProperty(PropertyId property, String value) { - m_properties.put(property, value); - } - - @Override - public void setProperties(Map mapProperties) { - m_properties.putAll(mapProperties); - } - - @Override - public Map getProperties() { - return m_properties; - } - @Override public boolean equals(Object o) { if (this == o) return true; Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ClusterResourceDefinition.java Wed Nov 7 08:27:36 2012 @@ -61,7 +61,7 @@ public class ClusterResourceDefinition e Resource.Type.Service).getKeyPropertyId(Resource.Type.Service)); mapChildren.put(serviceResource.getPluralName(), serviceResource); - HostResourceDefinition hostResource = new HostResourceDefinition(null, getId()); + HostResourceDefinition hostResource = new HostResourceDefinition(null, getId(), true); hostResource.getQuery().addProperty(getClusterController().getSchema( Resource.Type.Host).getKeyPropertyId(Resource.Type.Host)); mapChildren.put(hostResource.getPluralName(), hostResource); Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ComponentResourceDefinition.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ComponentResourceDefinition.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ComponentResourceDefinition.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ComponentResourceDefinition.java Wed Nov 7 08:27:36 2012 @@ -39,12 +39,6 @@ public class ComponentResourceDefinition private String m_clusterId; /** - * value of serviceId foreign key - */ - private String m_serviceId; - - - /** * Constructor. * * @param id value of component id @@ -54,9 +48,9 @@ public class ComponentResourceDefinition public ComponentResourceDefinition(String id, String clusterId, String serviceId) { super(Resource.Type.Component, id); m_clusterId = clusterId; - m_serviceId = serviceId; + setResourceId(Resource.Type.Cluster, m_clusterId); - setResourceId(Resource.Type.Service, m_serviceId); + setResourceId(Resource.Type.Service, serviceId); } @Override @@ -94,17 +88,17 @@ public class ComponentResourceDefinition } @Override - public void setParentId(Resource.Type type, String value) { - if (type == Resource.Type.HostComponent) { - setId(value); - } else { - super.setParentId(type, value); + public void setParentIds(Map mapIds) { + String id = mapIds.remove(Resource.Type.HostComponent); + if (id != null) { + setId(id); } + super.setParentIds(mapIds); } /** - * Base resource processor which generates href's. This is called by the {@link org.apache.ambari.server.api.services.ResultPostProcessor} during post - * processing of a result. + * Base resource processor which generates href's. This is called by the + * {@link org.apache.ambari.server.api.services.ResultPostProcessor} during post processing of a result. */ private class ComponentHrefProcessor extends BaseHrefPostProcessor { @Override @@ -113,12 +107,11 @@ public class ComponentResourceDefinition if (parent.getParent() != null && parent.getParent().getObject().getType() == Resource.Type.HostComponent) { Resource r = resultNode.getObject(); - String clusterId = getResourceIds().get(Resource.Type.Cluster); Schema schema = ClusterControllerHelper.getClusterController().getSchema(r.getType()); Object serviceId = r.getPropertyValue(schema.getKeyPropertyId(Resource.Type.Service)); Object componentId = r.getPropertyValue(schema.getKeyPropertyId(r.getType())); - href = href.substring(0, href.indexOf(clusterId) + clusterId.length() + 1) + + href = href.substring(0, href.indexOf("/hosts/") + 1) + "services/" + serviceId + "/components/" + componentId; resultNode.setProperty("href", href); Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ConfigurationResourceDefinition.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ConfigurationResourceDefinition.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ConfigurationResourceDefinition.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ConfigurationResourceDefinition.java Wed Nov 7 08:27:36 2012 @@ -49,9 +49,6 @@ public class ConfigurationResourceDefini super(Resource.Type.Configuration, configType); m_clusterId = clusterId; setResourceId(Resource.Type.Cluster, m_clusterId); - - if (null != configTag) - setProperty(PropertyHelper.getPropertyId("tag", "Config"), configTag); } @Override @@ -83,15 +80,17 @@ public class ConfigurationResourceDefini public void process(Request request, TreeNode resultNode, String href) { if (resultNode.getObject().getType() == Resource.Type.Configuration) { - String clusterId = getResourceIds().get(Resource.Type.Cluster); - String type = (String) resultNode.getObject().getPropertyValue(PropertyHelper.getPropertyId("type")); - String tag = (String) resultNode.getObject().getPropertyValue(PropertyHelper.getPropertyId("tag")); - if (! href.endsWith("/")) { href += '/'; } - href = href.substring(0, href.indexOf(clusterId) + clusterId.length() + 1) + - "configurations?type=" + type + "&tag=" + tag; + + String clustersToken = "/clusters"; + int idx = href.indexOf(clustersToken) + clustersToken.length() + 1; + idx = href.indexOf("/", idx) + 1; + + String type = (String) resultNode.getObject().getPropertyValue(PropertyHelper.getPropertyId("type")); + String tag = (String) resultNode.getObject().getPropertyValue(PropertyHelper.getPropertyId("tag")); + href = href.substring(0, idx) + "configurations?type=" + type + "&tag=" + tag; resultNode.setProperty("href", href); } else { Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostComponentResourceDefinition.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostComponentResourceDefinition.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostComponentResourceDefinition.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostComponentResourceDefinition.java Wed Nov 7 08:27:36 2012 @@ -38,11 +38,6 @@ public class HostComponentResourceDefini */ private String m_clusterId; - /** - * value of host id foreign key - */ - private String m_hostId; - /** * Constructor. @@ -54,9 +49,8 @@ public class HostComponentResourceDefini public HostComponentResourceDefinition(String id, String clusterId, String hostId) { super(Resource.Type.HostComponent, id); m_clusterId = clusterId; - m_hostId = hostId; setResourceId(Resource.Type.Cluster, m_clusterId); - setResourceId(Resource.Type.Host, m_hostId); + setResourceId(Resource.Type.Host, hostId); } @Override @@ -94,12 +88,12 @@ public class HostComponentResourceDefini } @Override - public void setParentId(Resource.Type type, String value) { - if (type == Resource.Type.Component) { - setId(value); - } else { - super.setParentId(type, value); + public void setParentIds(Map mapIds) { + String id = mapIds.remove(Resource.Type.Component); + if (id != null) { + setId(id); } + super.setParentIds(mapIds); } @@ -114,12 +108,11 @@ public class HostComponentResourceDefini if (parent.getParent() != null && parent.getParent().getObject().getType() == Resource.Type.Component) { Resource r = resultNode.getObject(); - String clusterId = getResourceIds().get(Resource.Type.Cluster); Schema schema = ClusterControllerHelper.getClusterController().getSchema(r.getType()); Object host = r.getPropertyValue(schema.getKeyPropertyId(Resource.Type.Host)); Object hostComponent = r.getPropertyValue(schema.getKeyPropertyId(r.getType())); - href = href.substring(0, href.indexOf(clusterId) + clusterId.length() + 1) + + href = href.substring(0, href.indexOf("/services/") + 1) + "hosts/" + host + "/host_components/" + hostComponent; resultNode.setProperty("href", href); Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostResourceDefinition.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostResourceDefinition.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostResourceDefinition.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/HostResourceDefinition.java Wed Nov 7 08:27:36 2012 @@ -37,13 +37,20 @@ public class HostResourceDefinition exte private String m_clusterId; /** + * Whether the host resource is associated with a cluster. + */ + private boolean m_attached; + + /** * Constructor. * * @param id host id value * @param clusterId cluster id value + * @param attached */ - public HostResourceDefinition(String id, String clusterId) { + public HostResourceDefinition(String id, String clusterId, boolean attached) { super(Resource.Type.Host, id); + m_attached = attached; m_clusterId = clusterId; setResourceId(Resource.Type.Cluster, m_clusterId); @@ -73,8 +80,7 @@ public class HostResourceDefinition exte public Map getSubResources() { Map mapChildren = new HashMap(); - // !!! is this a host for a cluster - if (null != m_clusterId) { + if (m_attached) { HostComponentResourceDefinition hostComponentResource = new HostComponentResourceDefinition(null, m_clusterId, getId()); hostComponentResource.getQuery().addProperty(getClusterController().getSchema( Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceDefinition.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceDefinition.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceDefinition.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/resources/ResourceDefinition.java Wed Nov 7 08:27:36 2012 @@ -63,12 +63,11 @@ public interface ResourceDefinition { public Resource.Type getType(); /** - * Set the value of the parent foreign key. + * Set the values of the parent foreign keys. * - * @param type resource type of the parent - * @param value vale of the parent id + * @param mapIds map of all parent foreign keys. Map from resource type to id value. */ - public void setParentId(Resource.Type type, String value); + public void setParentIds(Map mapIds); /** * Obtain the primary and foreign key properties for the resource. @@ -101,29 +100,6 @@ public interface ResourceDefinition { */ public List getPostProcessors(); - //TODO: refactor set/get Property methods out of this class - /** - * Set a property on this resource. - * - * @param property the property - * @param value the value - */ - public void setProperty(PropertyId property, String value); - - /** - * Set a map of properties on the resource. - * - * @param mapProperties a map of properties - */ - public void setProperties(Map mapProperties); - - /** - * Get the properties which have been set on this resource. - * - * @return the properties which have been set on this resource - */ - public Map getProperties(); - /** * Resource specific result processor. * Used to provide resource specific processing of a result. Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/BasePersistenceManager.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/BasePersistenceManager.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/BasePersistenceManager.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/BasePersistenceManager.java Wed Nov 7 08:27:36 2012 @@ -26,6 +26,7 @@ import org.apache.ambari.server.controll import java.util.Collections; import java.util.Map; +import java.util.Set; /** * Base PersistenceManager functionality. @@ -36,7 +37,7 @@ public abstract class BasePersistenceMan return ClusterControllerHelper.getClusterController(); } - protected Request createControllerRequest(Map properties) { - return PropertyHelper.getCreateRequest(Collections.singleton(properties)); + protected Request createControllerRequest(Set> setProperties) { + return PropertyHelper.getCreateRequest(setProperties); } } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/ComponentService.java Wed Nov 7 08:27:36 2012 @@ -84,6 +84,25 @@ public class ComponentService extends Ba } /** + * Handles: POST /clusters/{clusterID}/services/{serviceID}/components + * Create components by specifying an array of components in the http body. + * This is used to create multiple components in a single request. + * + * @param body http body + * @param headers http headers + * @param ui uri info + * + * @return status code only, 201 if successful + */ + @POST + @Produces("text/plain") + public Response createComponents(String body, @Context HttpHeaders headers, @Context UriInfo ui) { + + return handleRequest(headers, body, ui, Request.Type.POST, + createResourceDefinition(null, m_clusterName, m_serviceName)); + } + + /** * Handles: POST /clusters/{clusterID}/services/{serviceID}/components/{componentID} * Create a specific component. * Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/ConfigurationService.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/ConfigurationService.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/ConfigurationService.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/ConfigurationService.java Wed Nov 7 08:27:36 2012 @@ -59,11 +59,12 @@ public class ConfigurationService extend @Produces("text/plain") public Response getConfigurations(@Context HttpHeaders headers, @Context UriInfo ui) { return handleRequest(headers, null, ui, Request.Type.GET, - createResourceDefinition(null, null, m_clusterName)); + createResourceDefinition(null, m_clusterName)); } /** - * Handles URL: /clusters/{clusterId}/configurations. The body should contain: + * Handles URL: /clusters/{clusterId}/configurations + * The body should contain: * 
    * {
    *     "type":"type_string",
@@ -76,30 +77,30 @@ public class ConfigurationService extend
    *     }
    * }
    *
- * Get all services for a cluster. + * + * To create multiple configurations is a request, provide an array of configuration properties. * * @param headers http headers * @param ui uri info - * @return service collection resource representation + * @return status code only, 201 if successful */ @POST @Produces("text/plain") public Response createConfigurations(String body,@Context HttpHeaders headers, @Context UriInfo ui) { return handleRequest(headers, body, ui, Request.Type.POST, - createResourceDefinition(null, null, m_clusterName)); + createResourceDefinition(null, m_clusterName)); } /** * Create a service resource definition. * * @param configType configuration type - * @param configTag tag applied to the configuration * @param clusterName cluster name * * @return a service resource definition */ - ResourceDefinition createResourceDefinition(String configType, String configTag, String clusterName) { - return new ConfigurationResourceDefinition(configType, configTag, clusterName); + ResourceDefinition createResourceDefinition(String configType, String clusterName) { + return new ConfigurationResourceDefinition(configType, null, clusterName); } } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/CreatePersistenceManager.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/CreatePersistenceManager.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/CreatePersistenceManager.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/CreatePersistenceManager.java Wed Nov 7 08:27:36 2012 @@ -20,29 +20,38 @@ package org.apache.ambari.server.api.ser import org.apache.ambari.server.api.resources.ResourceDefinition; import org.apache.ambari.server.AmbariException; -import org.apache.ambari.server.controller.spi.ClusterController; -import org.apache.ambari.server.controller.spi.RequestStatus; -import org.apache.ambari.server.controller.spi.Resource; -import org.apache.ambari.server.controller.spi.Schema; +import org.apache.ambari.server.controller.spi.*; +import java.util.HashMap; import java.util.Map; +import java.util.Set; /** * Responsible for persisting the creation of a resource in the back end. */ public class CreatePersistenceManager extends BasePersistenceManager { @Override - public RequestStatus persist(ResourceDefinition resource) { + public RequestStatus persist(ResourceDefinition resource, Set> setProperties) { ClusterController controller = getClusterController(); Map mapResourceIds = resource.getResourceIds(); Resource.Type type = resource.getType(); Schema schema = controller.getSchema(type); - for (Map.Entry entry : mapResourceIds.entrySet()) { - resource.setProperty(schema.getKeyPropertyId(entry.getKey()), entry.getValue()); + if (setProperties.size() == 0) { + setProperties.add(new HashMap()); } + + for (Map mapProperties : setProperties) { + for (Map.Entry entry : mapResourceIds.entrySet()) { + PropertyId property = schema.getKeyPropertyId(entry.getKey()); + if (! mapProperties.containsKey(property)) { + mapProperties.put(property, entry.getValue()); + } + } + } + try { - return controller.createResources(type, createControllerRequest(resource.getProperties())); + return controller.createResources(type, createControllerRequest(setProperties)); } catch (AmbariException e) { //todo: handle exception throw new RuntimeException("Create of resource failed: " + e, e); Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/DeletePersistenceManager.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/DeletePersistenceManager.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/DeletePersistenceManager.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/DeletePersistenceManager.java Wed Nov 7 08:27:36 2012 @@ -20,15 +20,19 @@ package org.apache.ambari.server.api.ser import org.apache.ambari.server.api.resources.ResourceDefinition; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.spi.PropertyId; import org.apache.ambari.server.controller.spi.RequestStatus; +import java.util.Map; +import java.util.Set; + /** * Responsible for persisting the deletion of a resource in the back end. */ public class DeletePersistenceManager extends BasePersistenceManager { @Override - public RequestStatus persist(ResourceDefinition resource) { + public RequestStatus persist(ResourceDefinition resource, Set> setProperties) { try { //todo: need to account for multiple resources and user predicate return getClusterController().deleteResources(resource.getType(), Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostComponentService.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostComponentService.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostComponentService.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostComponentService.java Wed Nov 7 08:27:36 2012 @@ -84,6 +84,25 @@ public class HostComponentService extend } /** + * Handles POST /clusters/{clusterID}/hosts/{hostID}/host_components + * Create host components by specifying an array of host components in the http body. + * This is used to create multiple host components in a single request. + * + * @param body http body + * @param headers http headers + * @param ui uri info + * + * @return status code only, 201 if successful + */ + @POST + @Produces("text/plain") + public Response createHostComponents(String body, @Context HttpHeaders headers, @Context UriInfo ui) { + + return handleRequest(headers, body, ui, Request.Type.POST, + createResourceDefinition(null, m_clusterName, m_hostName)); + } + + /** * Handles POST /clusters/{clusterID}/hosts/{hostID}/host_components/{hostComponentID} * Create a specific host_component. * Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostService.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostService.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostService.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/HostService.java Wed Nov 7 08:27:36 2012 @@ -76,7 +76,7 @@ public class HostService extends BaseSer @PathParam("hostName") String hostName) { return handleRequest(headers, null, ui, Request.Type.GET, - createResourceDefinition(hostName, m_clusterName)); + createResourceDefinition(hostName, m_clusterName, ui)); } /** @@ -90,7 +90,26 @@ public class HostService extends BaseSer @GET @Produces("text/plain") public Response getHosts(@Context HttpHeaders headers, @Context UriInfo ui) { - return handleRequest(headers, null, ui, Request.Type.GET, createResourceDefinition(null, m_clusterName)); + return handleRequest(headers, null, ui, Request.Type.GET, createResourceDefinition(null, m_clusterName, ui)); + } + + /** + * Handles POST /clusters/{clusterID}/hosts + * Create hosts by specifying an array of hosts in the http body. + * This is used to create multiple hosts in a single request. + * + * @param body http body + * @param headers http headers + * @param ui uri info + * + * @return status code only, 201 if successful + */ + @POST + @Produces("text/plain") + public Response createHosts(String body, @Context HttpHeaders headers, @Context UriInfo ui) { + + return handleRequest(headers, body, ui, Request.Type.POST, + createResourceDefinition(null, m_clusterName, ui)); } /** @@ -111,7 +130,7 @@ public class HostService extends BaseSer @PathParam("hostName") String hostName) { return handleRequest(headers, body, ui, Request.Type.POST, - createResourceDefinition(hostName, m_clusterName)); + createResourceDefinition(hostName, m_clusterName, ui)); } /** @@ -132,7 +151,7 @@ public class HostService extends BaseSer @PathParam("hostName") String hostName) { return handleRequest(headers, body, ui, Request.Type.PUT, - createResourceDefinition(hostName, m_clusterName)); + createResourceDefinition(hostName, m_clusterName, ui)); } /** @@ -150,7 +169,7 @@ public class HostService extends BaseSer public Response updateHosts(String body, @Context HttpHeaders headers, @Context UriInfo ui) { return handleRequest(headers, body, ui, Request.Type.PUT, - createResourceDefinition(null, m_clusterName)); + createResourceDefinition(null, m_clusterName, ui)); } /** @@ -170,7 +189,7 @@ public class HostService extends BaseSer @PathParam("hostName") String hostName) { return handleRequest(headers, null, ui, Request.Type.DELETE, - createResourceDefinition(hostName, m_clusterName)); + createResourceDefinition(hostName, m_clusterName, ui)); } /** @@ -187,11 +206,12 @@ public class HostService extends BaseSer /** * Create a host resource definition. * - * @param hostName host name - * @param clusterName cluster name + * @param hostName host name + * @param clusterName cluster name + * @param ui uri information * @return a host resource definition */ - ResourceDefinition createResourceDefinition(String hostName, String clusterName) { - return new HostResourceDefinition(hostName, clusterName); + ResourceDefinition createResourceDefinition(String hostName, String clusterName, UriInfo ui) { + return new HostResourceDefinition(hostName, clusterName, ui.getRequestUri().toString().contains("/clusters/")); } } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/PersistKeyValueService.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/PersistKeyValueService.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/PersistKeyValueService.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/PersistKeyValueService.java Wed Nov 7 08:27:36 2012 @@ -21,14 +21,12 @@ package org.apache.ambari.server.api.ser import java.io.IOException; import java.util.Map; -import javax.servlet.http.HttpServletRequest; import javax.ws.rs.GET; import javax.ws.rs.POST; import javax.ws.rs.Path; import javax.ws.rs.PathParam; import javax.ws.rs.Produces; import javax.ws.rs.WebApplicationException; -import javax.ws.rs.core.Context; import javax.ws.rs.core.Response; import javax.xml.bind.JAXBException; @@ -36,8 +34,6 @@ import org.apache.ambari.server.state.fs import org.apache.ambari.server.utils.StageUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.codehaus.jackson.JsonGenerationException; -import org.codehaus.jackson.map.JsonMappingException; import com.google.inject.Inject; @@ -53,10 +49,9 @@ public class PersistKeyValueService { @POST @Produces("text/plain") - public Response update(String keyValues, - @Context HttpServletRequest req) + public Response update(String keyValues) throws WebApplicationException, InvalidStateTransitionException, - JsonGenerationException, JsonMappingException, JAXBException, IOException { + JAXBException, IOException { LOG.info("Received message from UI " + keyValues); Map keyValuesMap = StageUtils.fromJson(keyValues, Map.class); /* Call into the heartbeat handler */ @@ -77,8 +72,7 @@ public class PersistKeyValueService { @GET @Produces("text/plain") - public String getAllKeyValues() throws JsonGenerationException, - JsonMappingException, JAXBException, IOException { + public String getAllKeyValues() throws JAXBException, IOException { Map ret = persistKeyVal.getAllKeyValues(); String stringRet = StageUtils.jaxbToString(ret); LOG.info("Returning " + stringRet); Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/PersistenceManager.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/PersistenceManager.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/PersistenceManager.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/PersistenceManager.java Wed Nov 7 08:27:36 2012 @@ -19,8 +19,12 @@ package org.apache.ambari.server.api.services; import org.apache.ambari.server.api.resources.ResourceDefinition; +import org.apache.ambari.server.controller.spi.PropertyId; import org.apache.ambari.server.controller.spi.RequestStatus; +import java.util.Map; +import java.util.Set; + /** * Persistence manager which is responsible for persisting a resource state to the back end. * This includes create, update and delete operations. @@ -29,10 +33,12 @@ public interface PersistenceManager { /** * Persist a resource to the back end. * - * @param resource the resource to persist + * + * @param resource resource definition for request + * @param setProperties properties to be persisted. * * @return the request state. * */ - public RequestStatus persist(ResourceDefinition resource); + public RequestStatus persist(ResourceDefinition resource, Set> setProperties); } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/Request.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/Request.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/Request.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/Request.java Wed Nov 7 08:27:36 2012 @@ -26,6 +26,7 @@ import org.apache.ambari.server.controll import java.util.List; import java.util.Map; +import java.util.Set; /** * Provides information on the current request. @@ -120,9 +121,9 @@ public interface Request { /** * Obtain the properties which have been parsed from the http body. * - * @return a map containing the properties contained in the http body + * @return a set of maps containing the properties contained in the http body */ - public Map getHttpBodyProperties(); + public Set> getHttpBodyProperties(); //TODO: refactor persistence mechanism /** Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/RequestImpl.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/RequestImpl.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/RequestImpl.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/RequestImpl.java Wed Nov 7 08:27:36 2012 @@ -194,7 +194,7 @@ public class RequestImpl implements Requ } @Override - public Map getHttpBodyProperties() { + public Set> getHttpBodyProperties() { return getHttpBodyParser().parse(getHttpBody()); } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/UpdatePersistenceManager.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/UpdatePersistenceManager.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/UpdatePersistenceManager.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/UpdatePersistenceManager.java Wed Nov 7 08:27:36 2012 @@ -21,18 +21,22 @@ package org.apache.ambari.server.api.ser import org.apache.ambari.server.api.resources.ResourceDefinition; import org.apache.ambari.server.AmbariException; +import org.apache.ambari.server.controller.spi.PropertyId; import org.apache.ambari.server.controller.spi.RequestStatus; +import java.util.Map; +import java.util.Set; + /** * Responsible for persisting the updating of a resource in the back end. */ public class UpdatePersistenceManager extends BasePersistenceManager { @Override - public RequestStatus persist(ResourceDefinition resource) { + public RequestStatus persist(ResourceDefinition resource, Set> setProperties) { try { return getClusterController().updateResources(resource.getType(), createControllerRequest( - resource.getProperties()), resource.getQuery().getInternalPredicate()); + setProperties), resource.getQuery().getInternalPredicate()); } catch (AmbariException e) { //todo: handle exception throw new RuntimeException("Update of resource failed: " + e, e); Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/JsonPropertyParser.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/JsonPropertyParser.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/JsonPropertyParser.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/JsonPropertyParser.java Wed Nov 7 08:27:36 2012 @@ -24,42 +24,51 @@ import org.codehaus.jackson.JsonNode; import org.codehaus.jackson.map.ObjectMapper; import java.io.IOException; -import java.util.HashMap; -import java.util.Iterator; -import java.util.Map; +import java.util.*; /** * JSON parser which parses a JSON string into a map of properties and values. */ public class JsonPropertyParser implements RequestBodyParser { - private Map m_properties = new HashMap(); + private Set> m_setProperties = new HashSet>(); + @Override - public Map parse(String s) { + public Set> parse(String s) { + ObjectMapper mapper = new ObjectMapper(); if (s != null && ! s.isEmpty()) { + s = ensureArrayFormat(s); try { - processNode(mapper.readValue(s, JsonNode.class), ""); + JsonNode[] nodes = mapper.readValue(s, JsonNode[].class); + for(JsonNode node : nodes) { + Map mapProperties = new HashMap(); + processNode(node, "", mapProperties); + m_setProperties.add(mapProperties); + } } catch (IOException e) { throw new RuntimeException("Unable to parse json: " + e, e); } } - - return m_properties; + return m_setProperties; } - private void processNode(JsonNode node, String path) { + private void processNode(JsonNode node, String path, Map mapProperties) { Iterator iter = node.getFieldNames(); String name; while (iter.hasNext()) { name = iter.next(); JsonNode child = node.get(name); if (child.isContainerNode()) { - processNode(child, path.isEmpty() ? name : path + '.' + name); + processNode(child, path.isEmpty() ? name : path + '.' + name, mapProperties); } else { - m_properties.put(PropertyHelper.getPropertyId(name, path), child.asText()); + mapProperties.put(PropertyHelper.getPropertyId(name, path), child.asText()); } } } + + private String ensureArrayFormat(String s) { + return s.startsWith("[") ? s : '[' + s + ']'; + } } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/RequestBodyParser.java URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/RequestBodyParser.java?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/RequestBodyParser.java (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/java/org/apache/ambari/server/api/services/parsers/RequestBodyParser.java Wed Nov 7 08:27:36 2012 @@ -21,6 +21,7 @@ package org.apache.ambari.server.api.ser import org.apache.ambari.server.controller.spi.PropertyId; import java.util.Map; +import java.util.Set; /** * Parse the provided String into a map of properties and associated values. @@ -32,7 +33,7 @@ public interface RequestBodyParser { * * @param s the string body to be parsed * - * @return a map of properties or an empty map if no properties exist + * @return a set of maps of properties or an empty set if no properties exist */ - public Map parse(String s); + public Set> parse(String s); } Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/ambari-server.py URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/ambari-server.py?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/ambari-server.py (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/ambari-server.py Wed Nov 7 08:27:36 2012 @@ -41,6 +41,8 @@ IP_TBLS_ENABLED="Firewall is running" IP_TBLS_DISABLED="Firewall is not running" IP_TBLS_SRVC_NT_FND="iptables: unrecognized service" SERVER_START_CMD="{0}" + os.sep + "bin" + os.sep + "java -cp {1}"+ os.pathsep + ".." + os.sep + "lib" + os.sep + "ambari-server" + os.sep + "* org.apache.ambari.server.controller.AmbariServer" +# uncomment for debug +# SERVER_START_CMD="{0}" + os.sep + "bin" + os.sep + "java -Xdebug -Xrunjdwp:transport=dt_socket,address=5005,server=y,suspend=n -cp {1}"+ os.pathsep + ".." + os.sep + "lib" + os.sep + "ambari-server" + os.sep + "* org.apache.ambari.server.controller.AmbariServer" AMBARI_CONF_VAR="AMBARI_CONF_DIR" PG_ST_CMD = "service postgresql status" PG_START_CMD = "service postgresql start" Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/bootstrap.py URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/bootstrap.py?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/bootstrap.py (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/bootstrap.py Wed Nov 7 08:27:36 2012 @@ -64,10 +64,10 @@ class SCP(threading.Thread): class SSH(threading.Thread): """ Ssh implementation of this """ - def __init__(self, sshKeyFile, host, commands): + def __init__(self, sshKeyFile, host, command): self.sshKeyFile = sshKeyFile self.host = host - self.commands = commands + self.command = command self.ret = {"exitstatus" : -1, "log": "FAILED"} threading.Thread.__init__(self) pass @@ -81,7 +81,7 @@ class SSH(threading.Thread): def run(self): sshcommand = ["ssh", "-o", "ConnectTimeOut=3", "-o", "StrictHostKeyChecking=no", "-i", self.sshKeyFile, - self.host, ";".join(self.commands)] + self.host, self.command] sshstat = subprocess.Popen(sshcommand, stdout=subprocess.PIPE, stderr=subprocess.PIPE) log = sshstat.communicate() @@ -96,10 +96,10 @@ def splitlist(hosts, n): class PSSH: """Run SSH in parallel for a given list of hosts""" - def __init__(self, hosts, sshKeyFile, commands): + def __init__(self, hosts, sshKeyFile, command): self.hosts = hosts self.sshKeyFile = sshKeyFile - self.commands = commands + self.command = command self.ret = {} pass @@ -112,7 +112,7 @@ class PSSH: for chunk in splitlist(self.hosts, 20): chunkstats = [] for host in chunk: - ssh = SSH(self.sshKeyFile, host, self.commands) + ssh = SSH(self.sshKeyFile, host, self.command) ssh.start() chunkstats.append(ssh) pass @@ -173,32 +173,41 @@ class BootStrap: return os.path.join(self.scriptDir, "setupAgent.py") def runSetupAgent(self): - commands = ["export AMBARI_PASSPHRASE=" + os.environ[AMBARI_PASSPHRASE_VAR], "/tmp/setupAgent.py"] - pssh = PSSH(self.hostlist, self.sshkeyFile, commands) + command = "python /tmp/setupAgent.py " + os.environ[AMBARI_PASSPHRASE_VAR] + pssh = PSSH(self.hostlist, self.sshkeyFile, command) pssh.run() out = pssh.getstatus() logging.info("Parallel ssh returns " + pprint.pformat(out)) - - """ Test code for setting env var on agent host before starting setupAgent.py - commands = ["export AMBARI_PASSPHRASE=" + os.environ[AMBARI_PASSPHRASE_VAR], "set"] - pssh = PSSH(self.hostlist, self.sshkeyFile, commands) - pssh.run() - out = pssh.getstatus() - logging.info("Look for AMBARI_PASSPHRASE in out " + pprint.pformat(out)) - """ + pass def copyNeededFiles(self): try: """Copying the files """ + """ Uncomment when ambari.repo is ready fileToCopy = self.getRepoFile() pscp = PSCP(self.hostlist, self.sshkeyFile, fileToCopy, "/etc/yum.repos.d") pscp.run() out = pscp.getstatus() logging.info("Parallel scp return " + pprint.pformat(out)) + """ + + """ Remove this block when ambari.repo is ready """ + """ copy agent rpm to remote host """ + pscp = PSCP(self.hostlist, self.sshkeyFile, "/home/centos/ambari/ambari-agent/target/rpm/ambari-agent/RPMS/noarch/ambari-agent*.rpm", "/tmp") + pscp.run() + out = pscp.getstatus() + logging.info("Parallel scp return " + pprint.pformat(out)) + + + pscp = PSCP(self.hostlist, self.sshkeyFile, "/usr/lib/python2.6/site-packages/ambari_server/setupAgent.py", "/tmp") + pscp.run() + out = pscp.getstatus() + logging.info("Parallel scp return " + pprint.pformat(out)) + except Exception as e: logging.info("Traceback " + traceback.format_exc()) pass - + pass def run(self): Modified: incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/setupAgent.py URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/setupAgent.py?rev=1406492&r1=1406491&r2=1406492&view=diff ============================================================================== --- incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/setupAgent.py (original) +++ incubator/ambari/branches/AMBARI-666/ambari-server/src/main/python/setupAgent.py Wed Nov 7 08:27:36 2012 @@ -29,26 +29,57 @@ import threading import traceback from pprint import pformat -def installAgent(): - """ Run yum install and make sure the agent install alright """ - # TODO replace echo with yum - yumcommand = ["echo", "install", "ambari-agent"] - yumstat = subprocess.Popen(yumcommand, stdout=subprocess.PIPE) - log = yumstat.communicate(0) +AMBARI_PASSPHRASE_VAR = "AMBARI_PASSPHRASE" + +def execOsCommand(osCommand): + """ Run yum install and make sure the puppet install alright """ + osStat = subprocess.Popen(osCommand, stdout=subprocess.PIPE) + log = osStat.communicate(0) ret = {} - ret["exitstatus"] = yumstat.returncode + ret["exitstatus"] = osStat.returncode ret["log"] = log return ret +def installPreReq(): + """ Adds hdp repo + rpmCommand = ["rpm", "-Uvh", "http://public-repo-1.hortonworks.com/HDP-1.1.1.16/repos/centos6/hdp-release-1.1.1.16-1.el6.noarch.rpm"] + execOsCommand(rpmCommand) + """ + yumCommand = ["yum", "-y", "install", "epel-release"] + execOsCommand(yumCommand) + +def installPuppet(): + """ Run yum install and make sure the puppet install alright """ + osCommand = ["useradd", "-G", "puppet", "puppet"] + execOsCommand(osCommand) + yumCommand = ["yum", "-y", "install", "puppet"] + return execOsCommand(yumCommand) + +def installAgent(): + """ Run yum install and make sure the agent install alright """ + # TODO replace rpm with yum -y + rpmCommand = ["yum", "install", "-y", "/tmp/ambari-agent*.rpm"] + return execOsCommand(rpmCommand) + def configureAgent(): """ Configure the agent so that it has all the configs knobs properly installed """ return +def runAgent(passPhrase): + os.environ[AMBARI_PASSPHRASE_VAR] = passPhrase + subprocess.call("/usr/sbin/ambari-agent start", shell=True) + def main(argv=None): scriptDir = os.path.realpath(os.path.dirname(argv[0])) + """ Parse the input""" + onlyargs = argv[1:] + passPhrase = onlyargs[0] + installPreReq() + # installPuppet() installAgent() configureAgent() + runAgent(passPhrase) if __name__ == '__main__': logging.basicConfig(level=logging.DEBUG)