incubator-ambari-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From maha...@apache.org
Subject svn commit: r1395859 - in /incubator/ambari/branches/AMBARI-666: ./ ambari-agent/src/main/python/ambari_agent/ ambari-agent/src/main/python/manifestGenerator/
Date Tue, 09 Oct 2012 05:06:15 GMT
Author: mahadev
Date: Tue Oct  9 05:06:15 2012
New Revision: 1395859

URL: http://svn.apache.org/viewvc?rev=1395859&view=rev
Log:
AMBARI-831. Move manifest generation into the ambari agent directory. (mahadev)

Added:
    incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/imports.txt
    incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
    incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/rolesToClass.dict
    incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/serviceStates.dict
    incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp
    incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json
Removed:
    incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/manifestGenerator/imports.txt
    incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/manifestGenerator/manifestGenerator.py
    incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/manifestGenerator/rolesToClass.dict
    incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/manifestGenerator/serviceStates.dict
    incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/manifestGenerator/test.json
Modified:
    incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt

Modified: incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt?rev=1395859&r1=1395858&r2=1395859&view=diff
==============================================================================
--- incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt (original)
+++ incubator/ambari/branches/AMBARI-666/AMBARI-666-CHANGES.txt Tue Oct  9 05:06:15 2012
@@ -12,6 +12,9 @@ AMBARI-666 branch (unreleased changes)
 
   NEW FEATURES
 
+  AMBARI-831. Move manifest generation into the ambari agent directory.
+  (mahadev)
+
   AMBARI-828. Manifest generation for various actions from the server.
   (mahadev)
 

Added: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/imports.txt
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/imports.txt?rev=1395859&view=auto
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/imports.txt
(added)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/imports.txt
Tue Oct  9 05:06:15 2012
@@ -0,0 +1,12 @@
+hdp/manifests/*.pp
+hdp-hadoop/manifests/*.pp
+hdp-hbase/manifests/*.pp
+hdp-zookeeper/manifests/*.pp
+hdp-oozie/manifests/*.pp
+hdp-pig/manifests/*.pp
+hdp-sqoop/manifests/*.pp
+hdp-templeton/manifests/*.pp
+hdp-hive/manifests/*.pp
+hdp-hcat/manifests/*.pp
+hdp-mysql/manifests/*.pp
+hdp-monitor-webserver/manifests/*.pp

Added: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py?rev=1395859&view=auto
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
(added)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/manifestGenerator.py
Tue Oct  9 05:06:15 2012
@@ -0,0 +1,184 @@
+#!/usr/bin/env python2.6
+
+'''
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+'''
+
+import json
+import os.path
+import logging
+
+logger = logging.getLogger()
+
+  #read static imports from file and write them to manifest
+def writeImports(outputFile, inputFileName='imports.txt'):
+  inputFile = open(inputFileName, 'r')
+  modulesdir = os.path.abspath(os.getcwd() + "../../../puppet/modules/")
+  logger.info("Modules dir is " + modulesdir)
+  for line in inputFile:
+    modulename = line.rstrip('\n')
+    line = "import " + "\"" + modulesdir + "/" + modulename + "\"\n"
+    outputFile.write(line)
+    
+  inputFile.close()
+
+def generateManifest(inputJsonStr):
+#reading json
+  parsedJson = json.loads(inputJsonStr)
+  hostname = parsedJson['hostname']
+  clusterHostInfo = parsedJson['clusterHostInfo']
+  params = parsedJson['params']
+  configurations = parsedJson['configurations']
+  #hostAttributes = parsedJson['hostAttributes']
+  roles = parsedJson['roleCommands']
+  
+#writing manifest
+  manifest = open('site.pp', 'w')
+
+  #writing imports from external static file
+  writeImports(manifest)
+  
+  #writing nodes
+  writeNodes(manifest, clusterHostInfo)
+  
+  #writing params from map
+  writeParams(manifest, params)
+  
+  #writing config maps
+  writeConfigurations(manifest, configurations)
+
+  #writing host attributes
+  #writeHostAttributes(manifest, hostAttributes)
+
+  #writing task definitions 
+  writeTasks(manifest, roles)
+     
+  manifest.close()
+    
+  
+  #read dictionary
+def readDict(file, separator='='):
+  result = dict()
+  
+  for line in file :
+    dictTuple = line.partition(separator)
+    result[dictTuple[0].strip()] = dictTuple[2].strip()
+  
+  return result
+  
+
+  #write nodes
+def writeNodes(outputFile, clusterHostInfo):
+  for node in clusterHostInfo.iterkeys():
+    outputFile.write('$' + node + '= [')
+    coma = ''
+    
+    for value in clusterHostInfo[node]:
+      outputFile.write(coma + '\'' + value + '\'')
+      coma = ', '
+
+    outputFile.write(']\n')
+
+#write params
+def writeParams(outputFile, params):
+  for param in params.iterkeys():
+    outputFile.write('$' +  param + '="' + params[param] + '"\n')
+
+#write host attributes
+def writeHostAttributes(outputFile, hostAttributes):
+  outputFile.write('$hostAttributes={\n')
+
+  coma = ''
+  for attribute in hostAttributes.iterkeys():
+    outputFile.write(coma + '"' +  attribute + '" => "{' + hostAttributes[attribute] +
'"}')
+    coma = ',\n'
+
+  outputFile.write('}\n')
+
+#write configurations
+def writeConfigurations(outputFile, configs):
+  outputFile.write('$configuration =  {\n')
+
+  for configName in configs.iterkeys():
+    outputFile.write(configName + '=> {\n')
+    config = configs[configName]
+    
+    coma = ''
+    for configParam in config.iterkeys():
+      outputFile.write(coma + '"' + configParam + '" => "' + config[configParam] + '"')
+      coma = ',\n'
+
+    outputFile.write('\n},\n')
+    
+  outputFile.write('\n}\n')
+
+#write node tasks
+def writeTasks(outputFile, roles):
+  #reading dictionaries
+  rolesToClassFile = open('rolesToClass.dict', 'r')
+  rolesToClass = readDict(rolesToClassFile)
+  rolesToClassFile.close()
+
+  serviceStatesFile =  open('serviceStates.dict', 'r')
+  serviceStates = readDict(serviceStatesFile)
+  serviceStatesFile.close()
+
+  outputFile.write('node /default/ {\n ')
+  writeStages(outputFile, len(roles))
+  stageNum = 1
+
+  for role in roles :
+    rolename = role['role']
+    command = role['cmd']
+    taskParams = role['roleParams']
+    taskParamsNormalized = normalizeTaskParams(taskParams)
+    taskParamsPostfix = ''
+    
+    if len(taskParamsNormalized) > 0 :
+      taskParamsPostfix = ', ' + taskParamsNormalized
+    
+    className = rolesToClass[rolename]
+    serviceState = serviceStates[command]
+    
+    outputFile.write('class {\'' + className + '\':' + ' stage => ' + str(stageNum) +

+                     ', service_state => ' + serviceState + taskParamsPostfix + '}\n')
+    stageNum = stageNum + 1
+  outputFile.write('}\n')
+def normalizeTaskParams(taskParams):
+  result = ''
+  coma = ''
+  
+  for paramName in taskParams.iterkeys():
+    result = coma + result + paramName + ' => ' + taskParams[paramName]
+    coma = ','
+    
+  return result
+  
+def writeStages(outputFile, numStages):
+  arrow = ''
+  
+  for i in range(numStages):
+    outputFile.write(arrow + 'stage{' + str(i) + ' :}')
+    arrow = ' -> '
+  
+  outputFile.write('\n')
+    
+logging.basicConfig(level=logging.DEBUG)    
+#test code
+jsonFile = open('test.json', 'r')
+jsonStr = jsonFile.read() 
+generateManifest(jsonStr)

Added: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/rolesToClass.dict
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/rolesToClass.dict?rev=1395859&view=auto
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/rolesToClass.dict
(added)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/rolesToClass.dict
Tue Oct  9 05:06:15 2012
@@ -0,0 +1,28 @@
+NAMENODE = hdp-hadoop::namenode
+DATANODE = hdp-hadoop::datanode
+SNAMENODE = hdp-hadoop::snamenode
+JOBTRACKER = hdp-hadoop::jobtracker
+TASKTRACKER = hdp-hadoop::tasktracker
+HDFS_CLIENT = hdp-hadoop::client
+MAPREDUCE_CLIENT = hdp-hadoop::client
+ZOOKEEPER_SERVER = hdp-zookeeper
+ZOOKEEPER_CLIENT = hdp-zookeeper::client
+HBASE_MASTER = hdp-hbase::master
+HBASE_REGIONSERVER = hdp-hbase::regionserver
+HBASE_CLIENT = hdp-hbase::client
+PIG_CLIENT = hdp-pig
+SQOOP_CLIENT = hdp-sqoop
+OOZIE_SERVER = hdp-oozie::server
+OOZIE_CLIENT = hdp-oozie::client
+HIVE_CLIENT = hdp-hive::client
+HCATALOG_CLIENT = hdp-hcat
+HCATALOG_SERVER = hdp-hcat::server
+HIVE_SERVER = hdp-hive::server
+HIVE_MYSQL = hdp-mysql::server
+TEMPLETON_SERVER = hdp-templeton::server
+TEMPLETON_CLIENT = hdp-templeton::client
+DASHBOARD = hdp-dashboard
+NAGIOS_SERVER = hdp-nagios::server
+GANGLIA_MONITOR_SERVER = hdp-ganglia::server
+GANGLIA_MONITOR = hdp-ganglia::monitor
+HTTPD = hdp-monitor-webserver
\ No newline at end of file

Added: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/serviceStates.dict
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/serviceStates.dict?rev=1395859&view=auto
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/serviceStates.dict
(added)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/serviceStates.dict
Tue Oct  9 05:06:15 2012
@@ -0,0 +1,2 @@
+START = running
+INSTALL = installed_and_configured
\ No newline at end of file

Added: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp?rev=1395859&view=auto
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp
(added)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/site.pp
Tue Oct  9 05:06:15 2012
@@ -0,0 +1,31 @@
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hadoop/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hbase/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-zookeeper/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-oozie/manifests/*.pp""
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-pig/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-sqoop/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-templeton/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hive/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-hcat/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-mysql/manifests/*.pp"
+import "/Users/mahadev/workspace/ambari-workspace/ambari-git/ambari-agent/src/main/python/puppet/modules/hdp-monitor-webserver/manifests/*.pp"
+$NAMENODE= ['h2.hortonworks.com']
+$DATANODE= ['h1.hortonworks.com', 'h2.hortonworks.com']
+$hdfs_user="hdfs"
+$jdk_location="lah/blah"
+$configuration =  {
+$hdfs_site=> {
+"dfs.block.size" => "256000000",
+"dfs.replication" => "1"
+}
+$core_site=> {
+"fs.default.name" => "hrt8n36.cc1.ygridcore.net"
+}
+
+}
+node /default/ {
+ stage{0 :} -> stage{1 :}
+class {'hdp-hadoop::datanode': stage => 1, service_state => running}
+class {'hdp-hadoop::namenode': stage => 2, service_state => installed_and_configured}
+}

Added: incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json
URL: http://svn.apache.org/viewvc/incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json?rev=1395859&view=auto
==============================================================================
--- incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json
(added)
+++ incubator/ambari/branches/AMBARI-666/ambari-agent/src/main/python/ambari_agent/test.json
Tue Oct  9 05:06:15 2012
@@ -0,0 +1,32 @@
+{
+"commandId": "234",
+"hostname":  "h1.hortonworks.com",
+
+"clusterHostInfo" : 
+{
+"NAMENODE": ["h2.hortonworks.com"],
+"DATANODE": ["h1.hortonworks.com", "h2.hortonworks.com"]
+},
+"params": 
+{
+"hdfs_user" : "hdfs",
+"jdk_location": "lah/blah"
+},
+"configurations" : {
+"hdfs_site" : { "dfs.block.size" : "256000000", "dfs.replication" : "1" } ,
+"core_site" :  { "fs.default.name" : "hrt8n36.cc1.ygridcore.net" }
+},
+"roleCommands": [
+{
+"role" : "DATANODE", 
+"cmd": "START",
+"roleParams" : {
+}
+},
+{
+"role": "NAMENODE",
+"cmd": "INSTALL",
+"roleParams" : {}
+}
+]
+}



Mime
View raw message