incubator-tashi-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From strou...@apache.org
Subject svn commit: r1221528 - in /incubator/tashi/trunk: Makefile etc/Accounting.cfg etc/Agent.cfg etc/TashiDefaults.cfg src/tashi/accounting/ src/tashi/accounting/accounting.py src/tashi/clustermanager/clustermanagerservice.py
Date Tue, 20 Dec 2011 23:44:29 GMT
Author: stroucki
Date: Tue Dec 20 23:44:28 2011
New Revision: 1221528

URL: http://svn.apache.org/viewvc?rev=1221528&view=rev
Log:
accounting.py: Add separate optional accounting server
clustermanagerservice.py: Remove chatty accounting messages
configs: add sample configs for Agent and Accounting to log to separate files
Makefile: Add accounting server to Makefile

Added:
    incubator/tashi/trunk/etc/Accounting.cfg
    incubator/tashi/trunk/etc/Agent.cfg
    incubator/tashi/trunk/src/tashi/accounting/
    incubator/tashi/trunk/src/tashi/accounting/accounting.py   (with props)
Modified:
    incubator/tashi/trunk/Makefile
    incubator/tashi/trunk/etc/TashiDefaults.cfg
    incubator/tashi/trunk/src/tashi/clustermanager/clustermanagerservice.py

Modified: incubator/tashi/trunk/Makefile
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/Makefile?rev=1221528&r1=1221527&r2=1221528&view=diff
==============================================================================
--- incubator/tashi/trunk/Makefile (original)
+++ incubator/tashi/trunk/Makefile Tue Dec 20 23:44:28 2011
@@ -65,15 +65,20 @@ src/utils/nmd: src/utils/nmd.py
 rmnmd:
 	echo Removing nmd...; rm -f bin/nmd.py
 
-bin: bindir bin/clustermanager.py bin/nodemanager.py bin/tashi-client.py bin/primitive.py
bin/zoni-cli.py
+bin: bindir bin/clustermanager.py bin/nodemanager.py bin/tashi-client.py bin/primitive.py
bin/zoni-cli.py bin/accounting.py
 bindir:
 	if test ! -d bin; then mkdir bin; fi
-rmbin: rmclustermanager rmnodemanager rmtashi-client rmprimitive rmzoni-cli
+rmbin: rmclustermanager rmnodemanager rmtashi-client rmprimitive rmzoni-cli rmaccounting
 	if test -d bin; then rmdir bin; fi
 bin/getInstances: 
 	if test ! -e bin/getInstances; then (echo "Generating client symlinks..."; cd bin; PYTHONPATH=../src
../src/tashi/client/client.py --makesyms); fi
 rmclients:
 	if test -e bin/getInstances; then (echo Removing client symlinks...; cd bin; PYTHONPATH=../src
../src/tashi/client/client.py --rmsyms; cd ..); fi
+bin/accounting.py: src/tashi/accounting/accounting.py
+	@echo Symlinking in Accounting server...
+	(cd bin; ln -s ../src/tashi/accounting/accounting.py .)
+rmaccounting:
+	if test -e bin/accounting.py; then echo Removing Accounting server symlink...; rm bin/accounting.py;
fi
 bin/clustermanager.py: src/tashi/clustermanager/clustermanager.py
 	@echo Symlinking in clustermanager...
 	(cd bin; ln -s ../src/tashi/clustermanager/clustermanager.py .)

Added: incubator/tashi/trunk/etc/Accounting.cfg
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/etc/Accounting.cfg?rev=1221528&view=auto
==============================================================================
--- incubator/tashi/trunk/etc/Accounting.cfg (added)
+++ incubator/tashi/trunk/etc/Accounting.cfg Tue Dec 20 23:44:28 2011
@@ -0,0 +1,18 @@
+[handler_fileHandler]
+class = FileHandler
+level = NOTSET
+formatter = standardFormatter
+args = ("/var/log/accounting.log",)
+
+[handlers]
+#keys = consoleHandler,publisherHandler,fileHandler
+keys = consoleHandler,fileHandler
+
+[loggers]
+keys = root
+
+[logger_root]
+level = DEBUG
+#handlers = consoleHandler,publisherHandler,fileHandler,syslogHandler
+handlers = fileHandler
+propagate = 1

Added: incubator/tashi/trunk/etc/Agent.cfg
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/etc/Agent.cfg?rev=1221528&view=auto
==============================================================================
--- incubator/tashi/trunk/etc/Agent.cfg (added)
+++ incubator/tashi/trunk/etc/Agent.cfg Tue Dec 20 23:44:28 2011
@@ -0,0 +1,18 @@
+[handler_fileHandler]
+class = FileHandler
+level = NOTSET
+formatter = standardFormatter
+args = ("/var/log/agent.log",)
+
+[handlers]
+#keys = consoleHandler,publisherHandler,fileHandler
+keys = consoleHandler,fileHandler
+
+[loggers]
+keys = root
+
+[logger_root]
+level = DEBUG
+#handlers = consoleHandler,publisherHandler,fileHandler,syslogHandler
+handlers = consoleHandler,fileHandler
+propagate = 1

Modified: incubator/tashi/trunk/etc/TashiDefaults.cfg
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/etc/TashiDefaults.cfg?rev=1221528&r1=1221527&r2=1221528&view=diff
==============================================================================
--- incubator/tashi/trunk/etc/TashiDefaults.cfg (original)
+++ incubator/tashi/trunk/etc/TashiDefaults.cfg Tue Dec 20 23:44:28 2011
@@ -87,6 +87,10 @@ file = /var/tmp/cm.dat
 uri = mysql://root@clustermanager/tashi
 password = changeme
 
+# Accounting portion
+[Accounting]
+publisher = tashi.messaging.GangliaPublisher
+
 # NodeManger portion
 [NodeManager]
 dfs = tashi.dfs.Vfs

Added: incubator/tashi/trunk/src/tashi/accounting/accounting.py
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/src/tashi/accounting/accounting.py?rev=1221528&view=auto
==============================================================================
--- incubator/tashi/trunk/src/tashi/accounting/accounting.py (added)
+++ incubator/tashi/trunk/src/tashi/accounting/accounting.py Tue Dec 20 23:44:28 2011
@@ -0,0 +1,76 @@
+#!/usr/bin/python
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+# 
+#   http://www.apache.org/licenses/LICENSE-2.0
+# 
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.    
+
+from socket import gethostname
+import os
+import socket
+import sys
+import threading
+import time
+import random
+import logging.config
+
+from tashi.rpycservices.rpyctypes import *
+from tashi.util import getConfig, createClient, instantiateImplementation, boolean
+import tashi
+
+class Accounting(object):
+	def __init__(self, config, cmclient):
+		self.config = config
+		self.cm = cmclient
+		self.hooks = []
+		self.log = logging.getLogger(__file__)
+
+		items = self.config.items("Accounting")
+		items.sort()
+		for item in items:
+			(name, value) = item
+			name = name.lower()
+			if (name.startswith("hook")):
+				try:
+					self.hooks.append(instantiateImplementation(value, config, cmclient, False))
+				except:
+					self.log.exception("Failed to load hook %s" % (value))
+					
+	def start(self):
+		while True:
+			try:
+				instances = self.cm.getInstances()
+				for instance in instances:
+					# XXXstroucki this currently duplicates what the CM was doing.
+					# perhaps implement a diff-like log?
+					self.log.info('Accounting: id %d host %d vmId %d user %d cores %d memory %d' % (instance.id,
instance.hostId, instance.vmId, instance.userId, instance.cores, instance.memory))
+			except:
+				self.log.warning("Accounting iteration failed")
+
+			# wait to do the next iteration
+			# XXXstroucki make this configurable?
+			time.sleep(60)
+
+def main():
+	(config, configFiles) = getConfig(["Accounting"])
+	publisher = instantiateImplementation(config.get("Accounting", "publisher"), config)
+	tashi.publisher = publisher
+	cmclient = createClient(config)
+	logging.config.fileConfig(configFiles)
+	accounting = Accounting(config, cmclient)
+	accounting.start()
+
+if __name__ == "__main__":
+	main()

Propchange: incubator/tashi/trunk/src/tashi/accounting/accounting.py
------------------------------------------------------------------------------
    svn:executable = *

Modified: incubator/tashi/trunk/src/tashi/clustermanager/clustermanagerservice.py
URL: http://svn.apache.org/viewvc/incubator/tashi/trunk/src/tashi/clustermanager/clustermanagerservice.py?rev=1221528&r1=1221527&r2=1221528&view=diff
==============================================================================
--- incubator/tashi/trunk/src/tashi/clustermanager/clustermanagerservice.py (original)
+++ incubator/tashi/trunk/src/tashi/clustermanager/clustermanagerservice.py Tue Dec 20 23:44:28
2011
@@ -236,6 +236,8 @@ class ClusterManagerService(object):
 			except:
 				self.log.exception('monitorCluster iteration failed')
 			#  XXXrgass too chatty.  Remove
+			# XXXstroucki the risk is that a deadlock in obtaining
+			# data could prevent this loop from continuing.
 			#self.log.info("Sleeping for %d seconds" % sleepFor)
 			time.sleep(sleepFor)
 
@@ -479,10 +481,8 @@ class ClusterManagerService(object):
 			oldHost.state = HostState.Normal
 
 		# let the host communicate what it is running
-		# XXXrgass - This is too chatty for the console, I think we should remove this.
-		# XXXstroucki - My install depends on this, but I output to log files. This should be handled
by a separate accounting server in future.
+		# and note that the information is not stale
 		for instance in instances:
-			self.log.info('Accounting: id %d host %d vmId %d user %d cores %d memory %d' % (instance.id,
host.id, instance.vmId, instance.userId, instance.cores, instance.memory))
 			self.instanceLastContactTime.setdefault(instance.id, 0)
 
 		self.data.releaseHost(oldHost)



Mime
View raw message