incubator-tashi-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mry...@apache.org
Subject svn commit: r710072 [1/3] - in /incubator/tashi/import: ./ tashi-intel-r399/ tashi-intel-r399/doc/ tashi-intel-r399/etc/ tashi-intel-r399/guest/ tashi-intel-r399/scripts/ tashi-intel-r399/src/ tashi-intel-r399/src/tashi/ tashi-intel-r399/src/tashi/agen...
Date Mon, 03 Nov 2008 14:45:27 GMT
Author: mryan3
Date: Mon Nov  3 06:45:25 2008
New Revision: 710072

URL: http://svn.apache.org/viewvc?rev=710072&view=rev
Log:
This is the initial import of Tashi into the repository.
I'm keeping it in this import folder until repository organization and other administrative matters are dealt with.


Added:
    incubator/tashi/import/
    incubator/tashi/import/tashi-intel-r399/
    incubator/tashi/import/tashi-intel-r399/.project
    incubator/tashi/import/tashi-intel-r399/.pydevproject
    incubator/tashi/import/tashi-intel-r399/Makefile
    incubator/tashi/import/tashi-intel-r399/README
    incubator/tashi/import/tashi-intel-r399/STYLE
    incubator/tashi/import/tashi-intel-r399/TODO
    incubator/tashi/import/tashi-intel-r399/doc/
    incubator/tashi/import/tashi-intel-r399/etc/
    incubator/tashi/import/tashi-intel-r399/etc/TashiDefaults.cfg
    incubator/tashi/import/tashi-intel-r399/etc/TestConfig.cfg
    incubator/tashi/import/tashi-intel-r399/guest/
    incubator/tashi/import/tashi-intel-r399/guest/tashi   (with props)
    incubator/tashi/import/tashi-intel-r399/scripts/
    incubator/tashi/import/tashi-intel-r399/scripts/create   (with props)
    incubator/tashi/import/tashi-intel-r399/scripts/demo-yahoo-08-14-08   (with props)
    incubator/tashi/import/tashi-intel-r399/scripts/mryan3-database-setup   (with props)
    incubator/tashi/import/tashi-intel-r399/scripts/resume   (with props)
    incubator/tashi/import/tashi-intel-r399/scripts/stress   (with props)
    incubator/tashi/import/tashi-intel-r399/scripts/tomer-database-setup   (with props)
    incubator/tashi/import/tashi-intel-r399/src/
    incubator/tashi/import/tashi-intel-r399/src/tashi/
    incubator/tashi/import/tashi-intel-r399/src/tashi/__init__.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/agents/
    incubator/tashi/import/tashi-intel-r399/src/tashi/agents/__init__.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/agents/examplepolicy.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/client/
    incubator/tashi/import/tashi-intel-r399/src/tashi/client/__init__.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/client/client.py   (with props)
    incubator/tashi/import/tashi-intel-r399/src/tashi/client/test.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/
    incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/__init__.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/clustermanager.py   (with props)
    incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/clustermanagerservice.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/
    incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/__init__.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/datainterface.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/fromconfig.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/pickled.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/sql.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/connectionmanager.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/dfs/
    incubator/tashi/import/tashi-intel-r399/src/tashi/dfs/__init__.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/dfs/dfsinterface.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/dfs/vfs.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/messaging/
    incubator/tashi/import/tashi-intel-r399/src/tashi/messaging/__init__.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/messaging/messageBroker.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/messaging/messaging.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/messaging/soapmessaging.py   (with props)
    incubator/tashi/import/tashi-intel-r399/src/tashi/messaging/tashimessaging.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/messaging/threadpool.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/messaging/thriftmessaging.py   (with props)
    incubator/tashi/import/tashi-intel-r399/src/tashi/nodemanager/
    incubator/tashi/import/tashi-intel-r399/src/tashi/nodemanager/__init__.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/nodemanager/nodemanager.py   (with props)
    incubator/tashi/import/tashi-intel-r399/src/tashi/nodemanager/nodemanagerservice.py   (with props)
    incubator/tashi/import/tashi-intel-r399/src/tashi/nodemanager/notification.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/nodemanager/vmcontrol/
    incubator/tashi/import/tashi-intel-r399/src/tashi/nodemanager/vmcontrol/__init__.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/nodemanager/vmcontrol/newxen.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/nodemanager/vmcontrol/qemu.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/nodemanager/vmcontrol/vmcontrolinterface.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/nodemanager/vmcontrol/xenpv.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/parallel.py
    incubator/tashi/import/tashi-intel-r399/src/tashi/thrift/
    incubator/tashi/import/tashi-intel-r399/src/tashi/thrift/build.py   (with props)
    incubator/tashi/import/tashi-intel-r399/src/tashi/thrift/messagingthrift.thrift
    incubator/tashi/import/tashi-intel-r399/src/tashi/thrift/services.thrift
    incubator/tashi/import/tashi-intel-r399/src/tashi/util.py
    incubator/tashi/import/tashi-intel-r399/src/utils/
    incubator/tashi/import/tashi-intel-r399/src/utils/Makefile
    incubator/tashi/import/tashi-intel-r399/src/utils/nmd.c
    incubator/tashi/import/tashi-intel-r399/svn-pull
    incubator/tashi/import/tashi-intel-r399/test/
    incubator/tashi/import/tashi-intel-r399/test/clustermanager-rpc-data-management/
    incubator/tashi/import/tashi-intel-r399/test/clustermanager-rpc-data-management/test
    incubator/tashi/import/tashi-intel-r399/test/runall   (with props)

Added: incubator/tashi/import/tashi-intel-r399/.project
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/.project?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/.project (added)
+++ incubator/tashi/import/tashi-intel-r399/.project Mon Nov  3 06:45:25 2008
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>tashi</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

Added: incubator/tashi/import/tashi-intel-r399/.pydevproject
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/.pydevproject?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/.pydevproject (added)
+++ incubator/tashi/import/tashi-intel-r399/.pydevproject Mon Nov  3 06:45:25 2008
@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?>
+
+<pydev_project>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.5</pydev_property>
+</pydev_project>

Added: incubator/tashi/import/tashi-intel-r399/Makefile
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/Makefile?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/Makefile (added)
+++ incubator/tashi/import/tashi-intel-r399/Makefile Mon Nov  3 06:45:25 2008
@@ -0,0 +1,69 @@
+# This Makefile doesn't build a binary, but sets up this folder after a fresh checkout
+
+# Setup
+.SILENT:
+
+# Explicit builds
+all: src/tashi/services bin src/utils/nmd
+	@echo Done
+
+mryan3: src/tashi/services bin src/utils/nmd src/tags doc/html
+	@echo Done
+
+doc: rmdoc doc/html
+	@echo Done
+
+clean: rmnmd rmbin rmtags rmservices rmdoc
+	if [ `find . -name "*.pyc" | wc -l` -gt 0 ]; then echo Removing python byte-code...; rm `find . -name "*.pyc"`; fi
+	if [ `find ./test -name "log.txt" | wc -l` -gt 0 ]; then echo Removing test logs...; rm `find ./test -name "log.txt"`; fi
+	@echo Done
+
+# Implicit builds
+src/utils/nmd: src/utils/Makefile src/utils/nmd.c
+	@echo Building nmd...
+	(cd src/utils; make)
+	ln -s src/utils/nmd/nmd ./bin/nmd
+
+rmnmd:
+	if test -e src/utils/nmd; then echo Removing nmd...; (cd src/utils; make clean); rm -f bin/nmd; fi
+
+src/tashi/services: src/tashi/thrift/services.thrift
+	@echo Building tashi.services...
+	(cd src/tashi/thrift; ./build.py)
+
+rmservices:
+	if test -d src/tashi/services; then echo Removing tashi.services...; rm -rf src/tashi/services; fi
+	if test -d src/tashi/thrift/gen-py; then echo Removing tashi.thrift.gen-py...; rm -rf src/tashi/thrift/gen-py; fi
+	if test -d src/tashi/messaging/messagingthrift; then echo Removing tashi.messaging.messagingthrift; rm -rf src/tashi/messaging/messagingthrift; fi
+
+bin: bindir bin/getInstances bin/clustermanager.py bin/nodemanager.py
+bindir:
+	if test ! -d bin; then mkdir bin; fi
+rmbin: rmclustermanager rmnodemanager rmclients
+	if test -d bin; then rmdir bin; fi
+bin/getInstances: src/tashi/services
+	if test ! -e bin/getInstances; then (echo "Generating client symlinks..."; cd bin; PYTHONPATH=../src ../src/tashi/client/client.py --makesyms); fi
+rmclients:
+	if test -e bin/getInstances; then (echo Removing client symlinks...; make src/tashi/services; cd bin; PYTHONPATH=../src ../src/tashi/client/client.py --rmsyms; cd ..); fi
+bin/clustermanager.py: src/tashi/clustermanager/clustermanager.py
+	@echo Symlinking in clustermanager...
+	(cd bin; ln -s ../src/tashi/clustermanager/clustermanager.py .)
+rmclustermanager:
+	if test -e bin/clustermanager.py; then echo Removing clustermanager symlink...; rm bin/clustermanager.py; fi
+bin/nodemanager.py: src/tashi/nodemanager/nodemanager.py
+	@echo Symlinking in nodemanager...
+	(cd bin; ln -s ../src/tashi/nodemanager/nodemanager.py .)
+rmnodemanager:
+	if test -e bin/nodemanager.py; then echo Removing nodemanager symlink...; rm bin/nodemanager.py; fi
+
+src/tags:
+	@echo Generating tags...
+	(cd src; ctags-exuberant -R --c++-kinds=+p --fields=+iaS --extra=+q -f ./tags .)
+rmtags:
+	if test -e src/tags; then echo Removing tags...; rm src/tags; fi
+
+doc/html:
+	@echo Generating HTML docs...
+	epydoc --html -o doc/html --include-log --name=tashi --graph=all --exclude=tashi.services --exclude=tashi.messaging.messagingthrift ./src/tashi
+rmdoc:
+	if test -d doc/html; then echo Removing HTML docs...; rm -rf ./doc/html; fi

Added: incubator/tashi/import/tashi-intel-r399/README
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/README?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/README (added)
+++ incubator/tashi/import/tashi-intel-r399/README Mon Nov  3 06:45:25 2008
@@ -0,0 +1,106 @@
+This is the Tashi package.
+
+Currently, we are using KVM and Xen.
+
+Quick start
+========================================================================================================================
+	XXX: This needs to be rewritten
+
+Notes on the VMs
+========================================================================================================================
+	KVM - uses Intel VT and is open source
+	KQEMU - syscalls faster than KVM, but everything else is slower
+	QEMU - same as KVM, but slower
+	Xen - crashed for me, but will try it again in the future
+	VMware - Not open-source -- does this exclude it?
+
+Filename				Description
+========================================================================================================================
+STYLE					Specifies some rules about what should and shouldn't be done to the code
+README					This file
+doc					Project documentation
+doc/external_OC2_pitch_04_03_08		First round of the external "OC2" pitch
+doc/reading_group_03_10_08		Reading group presentation
+doc/notes				Notes from project meetings
+doc/html				Automatically generated HTML doc for the project (made by mkhtmldoc.sh)
+mkhtmldoc.sh				Automatically generates HTML doc for the project
+.pydevproject				Eclipse project file?
+.project				Eclipse project file?
+TODO					List of things to do for the project
+src					Root of the python packages
+src/tashi				Base tashi package
+src/tashi/__init__.py			Contains some universally useful functions
+src/tashi/messaging			Messaging subsystem
+src/tashi/client			Client package
+src/tashi/client/client.py		Client executable
+src/tashi/client/__init__.py		Package stub
+src/tashi/data				Data backend package (for Cluster manager)
+src/tashi/data/__init__.py		Package functions
+src/tashi/data/schema.py		Database schema
+src/tashi/data/util.py			Utility functions
+src/tashi/services			Generated by tashi/thrift/build.py (thrift generated code)
+src/tashi/nodemanager			Node manager package -- needs to be reorganized
+src/tashi/thrift			Thrift stuff
+src/tashi/thrift/services.thrift	Thrift spec
+src/tashi/thrift/build.py		Tool to build the thrift code and put it in the right place
+src/tashi/clustermanager		Cluster manager package
+src/tashi/clustermanager/__init__.py	Cluster manager functions
+src/tashi/clustermanager/policies.py	Simple policy implementation (XXX: this needs to be reorganized)
+src/tashi/clustermanager/service.py	Service implemenation (for thrift RPCs)
+src/tashi/clustermanager/demo.py	Populate the data backend with test data
+src/tashi/clustermanager/clusterman...	Cluster manager executable
+etc					Configuration files
+etc/ClusterManager.cfg			Cluster manager configuration file
+etc/ClusterManagerLogging.cfg		Cluster manager logging configuration file (going away)
+guest					Guest stuff
+guest/tashi				Script for setting the hostname from the IP and registering the IP
+
+Client
+========================================================================================================================
+The client uses thrift RPCs to communicate with the Cluster Manager
+
+Guest
+========================================================================================================================
+Steps to setup a guest:
+	XXX: Optional
+	Remove /etc/hostname so that the hostname is not fixed
+	Place "oc2" script n /etc/network/if-up.d/ to set the hostname and register the IP with the master
+	Comment out eth0 in /etc/iftab so that multiple mac addresses show up as eth0
+	Add "acpi=force" to the kernel arguments to support shutdown
+	Add "noapictimer" if configuring a 64-bit guest
+	Install SSH so that the machine can be accessed
+
+NodeManager
+========================================================================================================================
+The steps currently involved in preping a machine to be a host include:
+	XXX: This list needs to be rewritten
+	# Enable VT in the BIOS (for Dell machines, "./tokenCtlS --token 0x014b --activate"), rebooting if necessary
+	# Install KVM ("cd /; tar xvjf kvm-60-bin.tar.bz2")
+	# Load the new kernel modules ("rmmod kvm; rmmod kvm-intel; depmod -a; modprobe kvm-intel")
+	# Make sure SDL is installed ("apt-get install libsdl1.2debian-oss")
+	# Make sure bridge-utils is installed ("apt-get install bridge-utils")
+	# Setup a bridge for the guests ("brctl addbr vmbr")
+	# Add a physical NIC to the bridge ("brctl addif vmbr eth1")
+	# Setup that physical NIC to be up and in promiscuous mode ("ifconfig eth1 0.0.0.0 up promisc")
+	# Setup the bridge to be up and in promiscuous mode ("ifconfig vmbr up promisc")
+	Make sure the disk images are available ("mkdir /mnt/mryan3; mount mryan3-d3:/export /mnt/mryan3")
+
+To prepare an image for booting natively on a host:
+	XXX: This also needs to be rewritten
+	Add losetup to the initrd in /sbin
+	Apply the diff "initrd-real-boot-diff.txt" to the initrd
+	Rebuild the initrd
+	Place the image at /x/hd.img on the host machine (this could be part of initrd)
+	Set the kernel parameters to "root=/dev/hda1 rw --"
+
+ClusterManager
+========================================================================================================================
+XXX: There is a server that runs here -- more doc needed later
+
+Packages
+========================================================================================================================
+Python [Python, does not affect code]
+KVM [GPL & LPGL, external binary -- shouldn't affect code]
+Xen [?, external binary or library?]
+SQLAlchemy [MIT]
+Thrift [ASL eventually?]

Added: incubator/tashi/import/tashi-intel-r399/STYLE
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/STYLE?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/STYLE (added)
+++ incubator/tashi/import/tashi-intel-r399/STYLE Mon Nov  3 06:45:25 2008
@@ -0,0 +1,28 @@
+This is not specifically about syntax, but includes some info about trunk 
+checkins in general.
+
+Rules
+================================================================================
+1. Use comments, and make them docstrings.  epydoc will automatically generate 
+HTML docs from this.
+2. Do not modify auto-generated code (thrift output specifically).
+3. If you're going to make a big change to the code (restructuring, replacing 
+core functionality, etc.), do it in a branch and do it in such a way that life 
+is not painful for the trunk maintainer.
+4. If you're going to make a big change to the code, only make one big change 
+at a time.
+5. You should not check in code you know doesn't work to trunk.
+6. Library dependencies should not require the newest releases of packages.  A 
+rule of thumb: if it doesn't work on something a year old (feisty at the time 
+of this writting), it doesn't count as working.
+7. When making a big change, if you would like it to eventually go into trunk 
+(and you should always want that -- this is an Apache project), make sure that 
+everybody who would care about what you're doing atleast gives some form of 
+approval.
+
+Testing
+================================================================================
+Currently, testing means testing the classes that implement an interface 
+(VmControlInterface, DFSInterface, etc) as well as the RPC servers (test every 
+RPC call).  Additionally, it means regression testing.  We are not currently 
+doing strict unit testing.

Added: incubator/tashi/import/tashi-intel-r399/TODO
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/TODO?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/TODO (added)
+++ incubator/tashi/import/tashi-intel-r399/TODO Mon Nov  3 06:45:25 2008
@@ -0,0 +1 @@
+XXX: I emptied this file because it was hopelessly out of date -- MPR

Added: incubator/tashi/import/tashi-intel-r399/etc/TashiDefaults.cfg
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/etc/TashiDefaults.cfg?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/etc/TashiDefaults.cfg (added)
+++ incubator/tashi/import/tashi-intel-r399/etc/TashiDefaults.cfg Mon Nov  3 06:45:25 2008
@@ -0,0 +1,88 @@
+# ClusterManager portion
+[ClusterManager]
+service = tashi.clustermanager.ClusterManagerService
+data = tashi.clustermanager.data.Pickled
+sqlLogging = 0
+nodeManagerPort = 9883
+
+[ClusterManagerService]
+convertExceptions = True
+port = 9882
+expireHostTime = 30.0
+allowDecayed = 30.0
+;bind = 0.0.0.0 ; not supported (Thrift is missing support to specify what to bind to!)
+
+[FromConfig]
+host1 = Host(d={'id':1,'name':'blade043'})
+host2 = Host(d={'id':2,'name':'blade044'})
+host3 = Host(d={'id':3,'name':'blade045'})
+host4 = Host(d={'id':4,'name':'blade074'})
+machineType1 = MachineType(d={'id':1,'name':'1c-512m','memory':512,'cores':1})
+network1 = Network(d={'id':1,'name':'global'})
+network2 = Network(d={'id':2,'name':'NAT'})
+user1 = User(d={'id':1,'name':'mryan3'})
+
+[Pickled]
+file = /var/tmp/cm.dat
+
+# NodeManger portion
+[NodeManager]
+Dfs = tashi.dfs.Vfs
+VmControl = tashi.nodemanager.vmcontrol.Qemu
+Service = tashi.nodemanager.NodeManagerService
+
+[NodeManagerService]
+convertExceptions = True
+port = 9883
+registerFrequency = 10.0
+infoFile = /var/tmp/nm.dat
+clusterManagerHost = fillmein
+clusterManagerPort = 9882
+;bind = 0.0.0.0 ; not supported (Thrift is missing support to specify what to bind to!)
+
+[Qemu]
+qemuBin = /usr/local/bin/qemu-system-x86_64
+infoDir = /var/tmp/VmControlQemu/
+pollDelay = 1.0
+migrationRetries = 10
+monitorTimeout = 60.0
+migrateTimeout = 300.0
+maxParallelMigrations = 10
+
+[XenPV]
+vmNamePrefix = tashi
+
+[Vfs]
+prefix = /var/tmp/
+
+# Logging stuff
+[loggers]
+keys = root	
+
+[handlers]
+keys = consoleHandler
+
+[formatters]
+keys = standardFormatter
+
+[logger_root]
+level = DEBUG
+handlers = consoleHandler
+propagate = 1
+	
+[handler_consoleHandler]
+class = StreamHandler
+level = NOTSET
+formatter = standardFormatter
+args = (sys.stdout,)
+
+[formatter_standardFormatter]
+format=%(asctime)s [%(name)s:%(levelname)s] %(message)s
+datefmt=
+class=logging.Formatter
+
+# Message Broker
+[MessageBroker]
+host = localhost
+port = 1717
+

Added: incubator/tashi/import/tashi-intel-r399/etc/TestConfig.cfg
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/etc/TestConfig.cfg?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/etc/TestConfig.cfg (added)
+++ incubator/tashi/import/tashi-intel-r399/etc/TestConfig.cfg Mon Nov  3 06:45:25 2008
@@ -0,0 +1,3 @@
+[MessageBroker]
+host = localhost
+port = 1717
\ No newline at end of file

Added: incubator/tashi/import/tashi-intel-r399/guest/tashi
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/guest/tashi?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/guest/tashi (added)
+++ incubator/tashi/import/tashi-intel-r399/guest/tashi Mon Nov  3 06:45:25 2008
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+if [ "$IFACE" != eth0 ]; then
+	exit 0
+fi
+
+MAC_ADDR=`ifconfig eth0 | grep "HWaddr" | awk '{print $5}'`
+IP=`ifconfig eth0 | grep "inet addr" | sed 's/:/ /' | awk '{print $3}'`
+HN=`echo vm-${IP} | sed 's/\./-/g'`
+
+hostname ${HN}
+wget -O- "http://optimus.irp-cluster/tashi/register-ip.php?mac=${MAC_ADDR}&ip=${IP}" > /dev/null 2>&1

Propchange: incubator/tashi/import/tashi-intel-r399/guest/tashi
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/tashi/import/tashi-intel-r399/scripts/create
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/scripts/create?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/scripts/create (added)
+++ incubator/tashi/import/tashi-intel-r399/scripts/create Mon Nov  3 06:45:25 2008
@@ -0,0 +1,3 @@
+#! /bin/bash
+
+./bin/createVm "Instance(d={'name':'foobar','type':$1,'disks':[DiskConfiguration(d={'uri':'hercules.qcow','persistent':False})],'nics':[NetworkConfiguration(d={'network':2,'mac':'52:54:00:00:10:$2'})], 'hints':{'display':'True'}})"

Propchange: incubator/tashi/import/tashi-intel-r399/scripts/create
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/tashi/import/tashi-intel-r399/scripts/demo-yahoo-08-14-08
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/scripts/demo-yahoo-08-14-08?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/scripts/demo-yahoo-08-14-08 (added)
+++ incubator/tashi/import/tashi-intel-r399/scripts/demo-yahoo-08-14-08 Mon Nov  3 06:45:25 2008
@@ -0,0 +1,92 @@
+#! /bin/bash
+
+export PYTHONPATH=`pwd`/src/
+
+NUM=10
+
+BIGNODE=172.16.250.254
+
+BASEID=0
+
+date
+
+echo "Creating small VMs..."
+for h in `seq -w 1 ${NUM}`; do
+	echo "Creating VM #${h}..."
+	INFO=`./create 1 $h 2>/dev/null | grep "[ \t]id:"`
+	if [[ ${BASEID} -eq 0 ]]; then
+		BASEID=`echo ${INFO} | sed 's/[^0-9]*\([0-9]*\).*/\1/'`
+	fi
+done
+echo "Small VMs created"
+echo "Waiting for startup..."
+while [[ true ]]; do
+	PASS="True"
+	for h in `cat ~/hosts/vms.txt`; do
+		HN=`ssh root@bd.${h} "hostname 2> /dev/null" 2> /dev/null | cut -c -2`
+		if [[ "${HN}" != "vm" ]]; then
+			PASS="False"
+		fi
+	done
+	if [[ "${PASS}" == "True" ]]; then
+		break
+	fi
+	sleep 1
+done
+echo "Waiting finished"
+echo "Starting work on small VMs..."
+for h in `cat ~/hosts/vms.txt`; do
+	ssh root@bd.${h} "./run > /dev/null 2>&1" > /dev/null 2>&1 &
+done
+echo "Creating large VM..."
+./create 2 11 > /dev/null 2>&1
+echo "Done creating large VM"
+while [[ true ]]; do
+	sleep 5
+	COUNT=0
+	for h in `cat ~/hosts/vms.txt`; do
+		CNT=`ssh root@bd.${h} "ls /x/mryan3/cvm-out/*/*.txt 2> /dev/null | wc -l 2> /dev/null" 2> /dev/null`
+		COUNT=$((COUNT+CNT))
+	done
+	echo "${COUNT}/64 work items completed..."
+	if [[ ${COUNT} -eq 64 ]]; then
+		break
+	fi
+done
+echo "Work on small VMs completed"
+for i in `seq 1 ${NUM}`; do
+	wait
+done
+echo "Collecting output from small VMs to the large VM..."
+ssh root@bd.${BIGNODE} ./gather > /dev/null 2>&1
+echo "Done collecting output"
+echo "Destroying small VMs..."
+for i in `seq 1 ${NUM}`; do
+	./bin/destroyVm $((i+BASEID-1)) > /dev/null 2>&1
+done
+echo "Done destroying small VMs"
+echo "Doing work on large VM..."
+ssh root@bd.${BIGNODE} ./build > /dev/null 2>&1 &
+while [[ true ]]; do
+	sleep 2
+	SIZE=`ssh root@bd.${BIGNODE} "du -hs ./out.e 2> /dev/null | awk '{print "'$1'"}' 2> /dev/null" 2> /dev/null`
+	echo "${SIZE}/154M output data generated..."
+	if [[ "${SIZE}" == "154M" ]]; then
+		break
+	fi
+done
+wait
+echo "Work on large VM complete"
+echo "Copying final output file to localhost..."
+scp root@bd.${BIGNODE}:./out.e /tmp/out.e
+echo "Copy complete"
+echo "Destroying large VM..."
+./bin/destroyVm $((11+BASEID-1)) > /dev/null 2>&1
+echo "Large VM destroyed"
+echo "Generating output image from etree..."
+(cd ~/local/src/mryan3/BigDatavis/src; ./util/draw_slice -d /tmp/out.e 0 0 0 512 0 0 0 512 0 512 512 /tmp/output.jpg > /dev/null 2>&1)
+echo "Image complete"
+
+date
+
+qiv /tmp/output.jpg

Propchange: incubator/tashi/import/tashi-intel-r399/scripts/demo-yahoo-08-14-08
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/tashi/import/tashi-intel-r399/scripts/mryan3-database-setup
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/scripts/mryan3-database-setup?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/scripts/mryan3-database-setup (added)
+++ incubator/tashi/import/tashi-intel-r399/scripts/mryan3-database-setup Mon Nov  3 06:45:25 2008
@@ -0,0 +1,11 @@
+#! /bin/sh
+
+./bin/addUser "User(d={'username':'mryan3'})"
+./bin/addPersistentImage "PersistentImage(d={'userId':1,'name':'i386-ubuntu.qcow'})"
+./bin/addInstanceConfiguration "InstanceConfiguration(d={'name':'i386-512','memory':512,'cores':1})"
+./bin/addHardDiskConfiguration "HardDiskConfiguration(d={'index':0,'persistentImageId':1,'persistent':False,'instanceConfigurationId':1})"
+./bin/addNetworkInterfaceConfiguration "NetworkInterfaceConfiguration(d={'index':0,'instanceConfigurationId':1})"
+./bin/addHost "Host(d={'hostname':'blade043'})"
+./bin/addHost "Host(d={'hostname':'blade044'})"
+./bin/addHost "Host(d={'hostname':'blade045'})"
+./bin/addHost "Host(d={'hostname':'blade074'})"

Propchange: incubator/tashi/import/tashi-intel-r399/scripts/mryan3-database-setup
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/tashi/import/tashi-intel-r399/scripts/resume
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/scripts/resume?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/scripts/resume (added)
+++ incubator/tashi/import/tashi-intel-r399/scripts/resume Mon Nov  3 06:45:25 2008
@@ -0,0 +1,3 @@
+#! /bin/bash
+
+./bin/resumeVm "Instance(d={'name':'foobar','type':$1,'disks':[DiskConfiguration(d={'uri':'i386-ubuntu.qcow','persistent':False})],'nics':[NetworkConfiguration(d={'network':2,'mac':'52:54:00:00:05:$2'})], 'hints':{'display':'True'}})" "\"$3\""

Propchange: incubator/tashi/import/tashi-intel-r399/scripts/resume
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/tashi/import/tashi-intel-r399/scripts/stress
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/scripts/stress?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/scripts/stress (added)
+++ incubator/tashi/import/tashi-intel-r399/scripts/stress Mon Nov  3 06:45:25 2008
@@ -0,0 +1,92 @@
+#! /bin/bash
+
+if [[ $# -eq 1 ]]; then
+	VMS=$1
+else
+	VMS=20
+fi
+
+date
+
+#HOSTS=`./bin/getHosts | grep name | wc -l`
+HOSTS="4"
+
+echo "Hosts: ${HOSTS}"
+
+echo "Create:"
+
+MID=10240000
+for i in `seq -w 1 $VMS`; do
+	echo "./scripts/create 2 $i"
+	ID=`./scripts/create 2 $i | grep "id: " | awk '{print $2}'`
+	if [[ ${ID} -lt ${MID} ]]; then
+		MID=${ID}
+	fi
+done
+
+date
+
+echo "Wait:"
+
+while [[ true ]]; do
+	CNT=`./bin/getInstances | grep -c Running`
+	echo ${CNT}
+	if [[ ${CNT} -eq ${VMS} ]]; then
+		break
+	fi
+	sleep 1
+done
+
+sleep 20
+
+date
+
+echo "Migrate:"
+
+for i in `seq 0 $((VMS-1))`; do
+	echo "./bin/migrateVm $((MID+i)) $((((i+1)%HOSTS)+1))"
+	./bin/migrateVm $((MID+i)) $((((i+1)%HOSTS)+1)) > /dev/null &
+done
+for i in `seq 0 $((VMS-1))`; do
+	wait
+done
+
+date
+
+echo "Wait:"
+
+while [[ true ]]; do
+	CNT=`./bin/getInstances | grep -c Running`
+	echo ${CNT}
+	if [[ ${CNT} -eq ${VMS} ]]; then
+		break
+	fi
+	sleep 1
+done
+
+date
+
+echo "Destroy:"
+
+for i in `seq 0 $((VMS-1))`; do
+	echo "./bin/destroyVm $((MID+i))"
+	./bin/destroyVm $((MID+i)) > /dev/null 2>&1 &
+done
+for i in `seq 0 $((VMS-1))`; do
+	wait
+done
+
+date
+
+echo "Wait:"
+
+while [[ true ]]; do
+	CNT=`./bin/getInstances | wc -l`
+	echo ${CNT}
+	if [[ ${CNT} -eq 1 ]]; then
+		break
+	fi
+	sleep 1
+done
+
+date

Propchange: incubator/tashi/import/tashi-intel-r399/scripts/stress
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/tashi/import/tashi-intel-r399/scripts/tomer-database-setup
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/scripts/tomer-database-setup?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/scripts/tomer-database-setup (added)
+++ incubator/tashi/import/tashi-intel-r399/scripts/tomer-database-setup Mon Nov  3 06:45:25 2008
@@ -0,0 +1,12 @@
+#! /bin/sh
+
+./bin/addUser "User(d={'username':'administrator'})"
+./bin/addUser "User(d={'username':'tshiran'})"
+./bin/addUser "User(d={'username':'jcipar'})"
+./bin/addPersistentImage "PersistentImage(d={'userId':1,'name':'hardy-25G.img'})"
+./bin/addPersistentImage "PersistentImage(d={'userId':1,'name':'hardy-25G-pdl.img','parentId':1})"
+./bin/addPersistentImage "PersistentImage(d={'userId':2,'name':'hardy-25G-tshiran.img','parentId':2})"
+./bin/addPersistentImage "PersistentImage(d={'userId':3,'name':'hardy-25G-jcipar.img','parentId':2})"
+./bin/addPersistentImage "PersistentImage(d={'userId':2,'name':'hardy-25G-tshiran-hadoop.img','parentId':3})"
+./bin/addHost "Host(d={'hostname':'ss306'})"
+./bin/addHost "Host(d={'hostname':'ss308'})"

Propchange: incubator/tashi/import/tashi-intel-r399/scripts/tomer-database-setup
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/__init__.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/__init__.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/__init__.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/__init__.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,2 @@
+from util import *
+from connectionmanager import ConnectionManager

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/agents/__init__.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/agents/__init__.py?rev=710072&view=auto
==============================================================================
    (empty)

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/agents/examplepolicy.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/agents/examplepolicy.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/agents/examplepolicy.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/agents/examplepolicy.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,80 @@
+from socket import gethostname
+import os
+import threading
+import time
+
+from tashi.services.ttypes import *
+from thrift.transport.TSocket import TServerSocket, TSocket
+from thrift.server.TServer import TThreadedServer
+from thrift.protocol.TBinaryProtocol import TBinaryProtocol
+from thrift.transport.TTransport import TBufferedTransport
+from tashi.services import clustermanagerservice
+
+class ExamplePolicy():
+	def __init__(self, client, transport):
+		self.client = client
+		self.transport = transport
+	
+	def start(self):
+		while True:
+			try:
+				if (not self.transport.isOpen()):
+					self.transport.open()
+				hosts = {}
+				load = {}
+				for h in self.client.getHosts():
+					hosts[h.id] = h
+					load[h.id] = []
+				load[None] = []
+				for i in self.client.getInstances():
+					if (i.hostId or i.state == InstanceState.Pending):
+						load[i.hostId] = load[i.hostId] + [i.id]
+				self.hosts = hosts
+				self.load = load
+				if (len(self.load.get(None, [])) > 0):
+					i = self.load[None][0]
+					min = None
+					minHost = None
+					for h in self.hosts.values():
+						if ((min is None or len(load[h.id]) < min) and h.up == True):
+							min = len(load[h.id])
+							minHost = h
+					if (minHost):
+						print "Scheduling instance %d on host %s" % (i, minHost.name)
+						self.client.activateVm(i, minHost)
+						continue
+				time.sleep(2)
+			except TashiException, e:
+				print e.msg
+				try:
+					self.transport.close()
+				except Exception, e:
+					print e
+				time.sleep(2)
+			except Exception, e:
+				print e
+				try:
+					self.transport.close()
+				except Exception, e:
+					print e
+				time.sleep(2)
+
+def createClient():
+	host = os.getenv('TASHI_CM_HOST', 'localhost')
+	port = os.getenv('TASHI_CM_PORT', '9882')
+	timeout = float(os.getenv('TASHI_CM_TIMEOUT', '5000.0'))
+	socket = TSocket(host, int(port))
+	socket.setTimeout(timeout)
+	transport = TBufferedTransport(socket)
+	protocol = TBinaryProtocol(transport)
+	client = clustermanagerservice.Client(protocol)
+	transport.open()
+	return (client, transport)
+
+def main():
+	(client, transport) = createClient()
+	agent = ExamplePolicy(client, transport)
+	agent.start()
+
+if __name__ == "__main__":
+	main()

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/client/__init__.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/client/__init__.py?rev=710072&view=auto
==============================================================================
    (empty)

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/client/client.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/client/client.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/client/client.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/client/client.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,168 @@
+#! /usr/bin/env python
+
+import inspect
+import os
+import sys
+from tashi.services.ttypes import *
+from thrift.protocol.TBinaryProtocol import TBinaryProtocol
+from thrift.transport.TTransport import TBufferedTransport
+from thrift.transport.TSocket import TSocket
+
+from tashi.services import clustermanagerservice
+from tashi import vmStates
+
+def makeHTMLTable(list):
+	(stdin_r, stdin_w) = os.pipe()
+	pipe = os.popen("tput cols")
+	columns = pipe.read().strip()
+	keys = {}
+	for k in list:
+		for k2 in k.__dict__.keys():
+			if (not k2.endswith("Obj")):
+				keys[k2] = k2
+	output = "<html>"
+	output = output + "<table>"
+	output = output + "<tr>"
+	for k in keys.keys():
+		output = output + "<td>%s</td>" % (k)
+	output = output + "</tr>"
+	for k in list:
+		output = output + "<tr>"
+		for k2 in keys.keys():
+			if (k2 == "state"):
+				output = output + "<td>%s</td>" % (str(vmStates[k.__dict__.get(k2, None)]))
+			else:
+				output = output + "<td>%s</td>" % (str(k.__dict__.get(k2, None)))
+		output = output + "</tr>"
+	output = output + "</table>"
+	output = output + "</html>"
+	pid = os.fork()
+	if (pid == 0):
+		os.close(stdin_w)
+		os.dup2(stdin_r, 0)
+		os.close(stdin_r)
+		os.execl("/usr/bin/lynx", "/usr/bin/lynx", "-width=%s" % (columns), "-dump", "-stdin")
+		sys.exit(-1)
+	os.close(stdin_r)
+	os.write(stdin_w, output)
+	os.close(stdin_w)
+	os.waitpid(pid, 0)
+
+def getFunction(argv):
+	"""Tries to determine the name of the function requested by the user -- may be called multiple times if the binary name is 'client'"""
+	function = "None"
+	if (len(argv) > 0):
+		function = argv[0].strip()
+		if (function.rfind("/") != -1):
+			function = function[function.rfind("/")+1:]
+		if (function.rfind(".") != -1):
+			function = function[:function.rfind(".")]
+	return function
+
+def getFunctionInfo(m):
+	"""Gets a string that describes a function from the interface"""
+	f = getattr(clustermanagerservice.Iface, m)
+	argspec = inspect.getargspec(f)[0][1:]
+	return m + inspect.formatargspec(argspec)
+
+def usage():
+	"""Print program usage"""
+	print "Available methods:"
+	for m in methods:
+		print "\t" + getFunctionInfo(m)
+	print
+	print "Examples:"
+	print "\tgetInstances"
+	print "\taddUser 'User(d={\"username\":\"foobar\"})'"
+	print "\tremoveUser 2"
+	print "\tcreateVM 1 1"
+
+def simpleType(obj):
+	"""Determines whether an object is a simple type -- used as a helper function to pprint"""
+	if (type(obj) is not type([])):
+		if (not getattr(obj, "__dict__", None)):
+			return True
+	return False
+
+def pprint(obj, depth = 0, key = None):
+	"""My own version of pprint that prints out a dict in a readable, but slightly more compact format"""
+	valueManip = lambda x: x
+	if (key):
+		keyString = key + ": "
+		if (key == "state"):
+			valueManip = lambda x: vmStates[x]
+	else:
+		keyString = ""
+	if (type(obj) is type([])):
+		if (reduce(lambda x, y: x and simpleType(y), obj, True)):
+			print (" " * (depth * INDENT)) + keyString + str(obj)
+		else:
+			print (" " * (depth * INDENT)) + keyString + "["
+			for o in obj:
+				pprint(o, depth + 1)
+			print (" " * (depth * INDENT)) + "]"
+	elif (getattr(obj, "__dict__", None)):
+		if (reduce(lambda x, y: x and simpleType(y), obj.__dict__.itervalues(), True)):
+			print (" " * (depth * INDENT)) + keyString + str(obj)
+		else:
+			print (" " * (depth * INDENT)) + keyString + "{"
+			for (k, v) in obj.__dict__.iteritems():
+				pprint(v, depth + 1, k)
+			print (" " * (depth * INDENT)) + "}"
+	else:
+			print (" " * (depth * INDENT)) + keyString + str(valueManip(obj))
+
+def main():
+	"""Main function for the client program"""
+	global INDENT, methods, exitCode
+	exitCode = 0
+	INDENT = (os.getenv("INDENT", 4))
+	methods = filter(lambda x: not x.startswith("__"), clustermanagerservice.Iface.__dict__.keys())
+	function = getFunction(sys.argv)
+	if (function == "client"):
+		function = getFunction(sys.argv[1:])
+	if (function == "--makesyms"):
+		for m in methods:
+			os.symlink(sys.argv[0], m)
+		sys.exit(0)
+	if (function == "--rmsyms"):
+		for m in methods:
+			os.unlink(m)
+		sys.exit(0)
+	host = os.getenv('TASHI_CM_HOST', 'localhost')
+	port = os.getenv('TASHI_CM_PORT', '9882')
+	timeout = float(os.getenv('TASHI_CM_TIMEOUT', '5000.0'))
+	socket = TSocket(host, int(port))
+	socket.setTimeout(timeout)
+	transport = TBufferedTransport(socket)
+	protocol = TBinaryProtocol(transport)
+	client = clustermanagerservice.Client(protocol)
+	client._transport = transport
+	client._transport.open()
+	f = getattr(client, function, None)
+	if not f:
+		usage()
+		sys.exit(-1)
+	args = map(lambda x: eval(x), sys.argv[1:])
+	try:
+		res = f(*args)
+		if (os.getenv("USE_HTML_TABLES")):
+			try:
+				makeHTMLTable(res)
+			except:
+				pprint(res)
+		else:
+			pprint(res)
+	except TashiException, e:
+		print e.msg
+		exitCode = e.errno
+	except TypeError, e:
+		print e
+		print "\t" + getFunctionInfo(function)
+		exitCode = -1
+	finally:
+		client._transport.close()
+	sys.exit(exitCode)
+
+if __name__ == "__main__":
+	main()

Propchange: incubator/tashi/import/tashi-intel-r399/src/tashi/client/client.py
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/client/test.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/client/test.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/client/test.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/client/test.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,297 @@
+import unittest
+import logging
+import sys
+import signal
+import os.path
+import copy
+import time
+import random
+from ConfigParser import ConfigParser
+
+from tashi.services.ttypes import *
+from thrift.transport.TSocket import TSocket
+from thrift.protocol.TBinaryProtocol import TBinaryProtocol
+from thrift.transport.TTransport import TBufferedTransport
+
+from tashi.services import clustermanagerservice
+from tashi.messaging.threadpool import synchronized
+from tashi.messaging.tashimessaging import TestTashiSubscriber
+
+from tashi.util import getConfig
+
+import tashi.client.client
+
+class ClientConnection():
+    '''Creates an rpc proxy'''
+    def __init__(self, host, port):
+	self.host = host
+	self.port = port
+	self.transport = TBufferedTransport(TSocket(host, int(port)))
+	self.protocol = TBinaryProtocol(self.transport)
+	self.client = clustermanagerservice.Client(self.protocol)
+	self.client._transport = self.transport
+	self.client._transport.open()
+    def __del__(self):
+        self.client._transport.close()
+
+def incrementor(init=0):
+    while 1:
+        yield init
+        init = init + 1
+
+# FIXME: don't duplicate code from clustermanager
+# def getConfig(args):
+#     config = ConfigParser()
+#     configFiles = [
+#        '/usr/share/tashi/ClusterManagerDefaults.cfg',
+#        '/etc/tashi/ClusterManager.cfg',
+#        os.path.expanduser('~/.tashi/ClusterManager.cfg')
+#        ] + ([args[0]] if len(args) > 0 else [])
+
+#     configFiles = config.read(configFiles)
+#     if len(configFiles) == 0:
+#         print >>sys.stderr, 'Unable to find the configuration file\n'
+#         sys.exit(3)
+
+#     return config
+
+
+class TestClient(unittest.TestCase):
+    @synchronized()
+    def getPortNum(self):
+        return self.portnum.next()
+
+    """macro test cases for single-host tests
+
+    Assumes cwd is 'src/tashi/client/'
+    """
+    def setUp(self):
+        """Create a CM and single NM on local host"""
+        logging.info('setting up test')
+        
+        (self.config, self.configfiles) = getConfig([])
+
+        self.port = 1717        # FIXME: take this (and other things) from config file
+        self.portnum = incrementor(self.port)
+
+        self.cwd = os.getcwd()
+        self.srcd = os.path.dirname(os.path.dirname(self.cwd))
+        
+        self.environ = copy.copy(os.environ)
+        self.environ['PYTHONPATH'] = self.srcd
+        logging.info('base path = %s' % self.srcd)
+
+        self.nm = os.spawnlpe(os.P_NOWAIT, 'python', 'python', 
+                              os.path.join(self.srcd, 'tashi', 'nodemanager', 'nodemanager.py'),
+                              self.environ)
+        self.cm = os.spawnlpe(os.P_WAIT, 'python', 'python',
+                             os.path.join(self.srcd,  'tashi', 'clustermanager', 'clustermanager.py'),
+                             '--drop', '--create',
+                             os.path.expanduser('~/.tashi/ClusterManager.cfg'),
+                             self.environ)
+        self.cm = os.spawnlpe(os.P_NOWAIT, 'python', 'python',
+                             os.path.join(self.srcd,  'tashi', 'clustermanager', 'clustermanager.py'),
+                             os.path.expanduser('~/.tashi/ClusterManager.cfg'),
+                             self.environ)
+        # since we are spawning with P_NOWAIT, we need to sleep to ensure that the CM is listening
+        time.sleep(1)
+        try:
+            self.connection = ClientConnection('localhost', self.config.get('ClusterManagerService', 'port'))
+        except Exception, e:
+            logging.warning('client connection failed')
+            ex = None
+            try:
+                logging.warning("setUp killing node manager " + str(self.nm))
+                os.kill(self.nm, signal.SIGKILL)
+            except Exception, e:
+                ex = e
+                logging.warning('could not kill node manager: '+ str(e))
+            try:
+                logging.warning('setUp killing cluster manager ' + str(self.cm))
+                os.kill(self.cm, signal.SIGKILL)
+            except Exception, e:
+                ex = e
+                logging.warning('could not kill cluster manager: ' + str(e))
+            if e != None:
+                raise e
+
+        logging.info('node manager PID: %i' % self.nm)
+    def tearDown(self):
+        '''Kill the CM and NM that were created by setUP'''
+        logging.info('tearing down test')
+        ex = None
+        try:
+            logging.debug("killing cluster manager " + str(self.cm))
+            os.kill(self.cm, signal.SIGKILL)
+        except Exception, e:
+            ex = e
+            logging.error('Could not kill cluster manager: ' + str(e))
+            
+        try:
+            logging.debug("killing node manager " + str(self.nm))
+            os.kill(self.nm, signal.SIGKILL)
+        except Exception, e:
+            ex = e
+            logging.error('Could not kill node manager: ' + str(e))
+        if ex != None:
+            raise ex
+    def testSetup(self):
+        '''empty test to ensure that setUp code works'''
+        logging.info('setting up')
+    def testHostManagement(self):
+        '''test adding/removing/listing hosts
+
+        Right now this just adds a single host: localhost.  Eventually
+        it should 1) take a list of hosts from a test configuration
+        file, 2) ensure that all were added, 3) remove a random
+        subset, 4) ensure that they were correctly removed, 5) remove
+        all, 6) ensure that they were correctly removed.'''
+
+        # get empty host list
+        hosts = self.connection.client.getHosts()
+        self.assertEqual(hosts, [], 'starting host list not empty: ' + str(hosts) )
+
+        # add a host
+        host = Host()
+        host.hostname = 'localhost'
+        host.enabled=True
+        self.connection.client.addHost(host)
+        hosts = self.connection.client.getHosts()
+        self.assertEqual(len(hosts), 1, 'wrong number of hosts %i, should be %i' % (len(hosts), 1) )
+        self.assertEqual(hosts[0].hostname, 'localhost', 'wrong hostname: ' + str(hosts[0].hostname) )
+
+        # remove first host
+        hid = hosts[0].id
+        self.connection.client.removeHost(hid)
+        hosts = self.connection.client.getHosts()
+        self.assertEqual(hosts, [], 'host list not empty after remove: ' + str(hosts) )
+
+    def testMessaging(self):
+        '''test messaging system started by CM
+
+        tests messages published directly, through events in the CM,
+        and the log system'''
+        # FIXME: add tests for generating events as a side-effect of
+        # rpc commands, as well as logging in the CM
+        portnum = self.getPortNum()
+        self.sub = TestTashiSubscriber(self.config, portnum)
+        self.assertEqual(self.sub.messageQueue.qsize(), 0)
+        self.pub = tashi.messaging.thriftmessaging.PublisherThrift(self.config.get('MessageBroker', 'host'),
+                                                                   int(self.config.get('MessageBroker', 'port')))
+        self.pub.publish({'message-type':'text', 'message':'Hello World!'})
+        time.sleep(0.5)
+        print '*** QSIZE', self.sub.messageQueue.qsize()
+        self.assertEqual(self.sub.messageQueue.qsize(), 1)
+
+        self.log = logging.getLogger(__name__)
+        messageHandler = tashi.messaging.tashimessaging.TashiLogHandler(self.config)
+        self.log.addHandler(messageHandler)
+        # FIXME: why can't we log messages with severity below 'warning'?
+        self.log.warning('test log message')
+        time.sleep(0.5)
+        self.assertEqual(self.sub.messageQueue.qsize(), 2)
+
+        # This should generate at least one log message
+#         hosts = self.connection.client.getHosts()
+#         time.sleep(0.5)
+#         if (self.sub.messageQueue.qsize() <= 2):
+#             self.fail()
+
+    def testUserManagement(self):
+        '''test adding/removing/listing users
+
+        same as testHostManagement, but with users'''
+        usernames = ['sleepy', 'sneezy', 'dopey', 'doc',
+                     'grumpy', 'bashful', 'happy']
+        # add all users
+        for un in usernames:
+            user = User()
+            user.username = un
+            self.connection.client.addUser(user)
+        # ensure that all were added
+        users = self.connection.client.getUsers()
+        self.assertEqual(len(usernames), len(users))
+        for user in users:
+            usernames.remove(user.username)
+        self.assertEqual(0, len(usernames))
+        # remove a random subset
+        rm = random.sample(users, 4)
+        for user in rm:
+            self.connection.client.removeUser(user.id)
+            users.remove(user)
+        newUsers = self.connection.client.getUsers()
+        # This ensures that the remaining ones are what we expect:
+        for user in newUsers:
+            # if there is a user remaining that we asked to be removed,
+            # this will throw an exception
+            users.remove(user)
+        # if a user was removed that we did not intend, this will
+        # throw an exception
+        self.assertEqual(0, len(users))
+
+#     def testInstanceConfigurationManagement(self):
+#         '''test adding/removing/listing instance configurations
+
+#         same as testHostManagement, but with instance configurations'''
+#         self.fail('test not implemented')
+    def testHardDiskConfigurationManagement(self):
+        '''test adding/removing/listing hard disk configurations
+
+        same as testHostManagement, but with hard disk configurations'''
+
+        user = User(d={'username':'sleepy'})
+        self.connection.client.addUser(user)
+        users = self.connection.client.getUsers()
+
+        per = PersistentImage()
+        per.userId = users[0].id
+        per.name = 'sleepy-PersistentImage'
+        self.connection.client.addPersistentImage(per)
+        pers = self.connection.client.getPersistentImages()
+
+        inst = InstanceConfiguration()
+        inst.name = 'sleepy-inst'
+        inst.memory = 512
+        inst.cores = 1
+        self.connection.client.addInstanceConfiguration(inst)
+        insts = self.connection.client.getInstanceConfigurations()
+
+        hdc = HardDiskConfiguration()
+        hdc.index = 0
+        hdc.persistentImageId = pers[0].id
+        hdc.persistent = False
+        hdc.instanceConfigurationId = insts[0].id
+
+#     def testCreateDestroyShutdown(self):
+#         '''test creating/destroying/shutting down VMs
+
+#         not implemented'''
+#         self.fail('test not implemented')
+#     def testSuspendResume(self):
+#         '''test suspending/resuming VMs
+
+#         not implemented'''
+#         self.fail('test not implemented')
+#     def testMigrate(self):
+#         '''test migration
+
+#         not implemented'''
+#         self.fail('test not implemented')
+#     def testPauseUnpause(self):
+#         '''test pausing/unpausing VMs
+
+#         not implemented'''
+#         self.fail('test not implemented')
+
+
+##############################
+# Test Code
+##############################
+if __name__ == '__main__':
+    logging.basicConfig(level=logging.NOTSET,
+                        format="%(asctime)s %(levelname)s:\t %(message)s",
+                        stream=sys.stdout)
+
+    suite = unittest.TestLoader().loadTestsFromTestCase(TestClient)
+    unittest.TextTestRunner(verbosity=2).run(suite)
+

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/__init__.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/__init__.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/__init__.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/__init__.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1 @@
+from clustermanagerservice import ClusterManagerService

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/clustermanager.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/clustermanager.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/clustermanager.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/clustermanager.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,59 @@
+#!/usr/bin/env python
+
+import os
+import sys
+import threading
+import signal
+import logging.config
+from getopt import getopt, GetoptError
+from ConfigParser import ConfigParser
+from thrift.transport.TSocket import TServerSocket
+from thrift.server.TServer import TThreadedServer
+
+from tashi.messaging.thriftmessaging import MessageBrokerThrift
+from tashi.messaging.tashimessaging import TashiLogHandler
+from tashi.services import clustermanagerservice
+from tashi.util import signalHandler, boolean, instantiateImplementation, getConfig, debugConsole
+
+def startClusterManager(config):
+    global service, data
+    
+    # start the event broker
+    broker = MessageBrokerThrift(int(config.get('MessageBroker', 'port')))
+    broker.ready.wait()
+    messageHandler = TashiLogHandler(config)
+    log.addHandler(messageHandler)
+
+    data = instantiateImplementation(config.get("ClusterManager", "data"), config)
+    service = instantiateImplementation(config.get("ClusterManager", "service"), config, data)
+    processor = clustermanagerservice.Processor(service)
+    transport = TServerSocket(int(config.get('ClusterManagerService', 'port')))
+    server = TThreadedServer(processor, transport)
+    
+    debugConsole(globals())
+    
+    try:
+        server.serve()
+    except KeyboardInterrupt:
+        handleSIGTERM(signal.SIGTERM, None)
+
+@signalHandler(signal.SIGTERM)
+def handleSIGTERM(signalNumber, stackFrame):
+    log.info('Exiting cluster manager after receiving a SIGINT signal')
+    sys.exit(0)
+    
+def main():
+    global log
+    
+    # setup configuration and logging
+    (config, configFiles) = getConfig(["ClusterManager"])
+    logging.config.fileConfig(configFiles)
+    log = logging.getLogger(__file__)
+    log.info('Using configuration file(s) %s' % configFiles)
+    
+    # bind the database
+    log.info('Starting cluster manager')
+    startClusterManager(config)
+
+if __name__ == "__main__":
+    main()

Propchange: incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/clustermanager.py
------------------------------------------------------------------------------
    svn:executable = *

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/clustermanagerservice.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/clustermanagerservice.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/clustermanagerservice.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/clustermanagerservice.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,446 @@
+from __future__ import with_statement
+
+from datetime import datetime
+from random import randint
+from socket import gethostname
+from thrift.transport.TSocket import TSocket
+from thrift.protocol.TBinaryProtocol import TBinaryProtocol
+from thrift.transport.TTransport import TBufferedTransport
+import logging
+import threading
+import time
+
+from tashi.messaging.thriftmessaging import MessageBrokerThrift
+from tashi.messaging.tashimessaging import TashiLogHandler
+from tashi.services.ttypes import Errors, InstanceState, HostState, TashiException
+from tashi.services import nodemanagerservice
+from tashi import boolean, convertExceptions, ConnectionManager, vmStates, timed
+
+def RPC(oldFunc):
+	return convertExceptions(oldFunc)
+
+class ClusterManagerService():
+	"""RPC service for the ClusterManager"""
+	
+	def __init__(self, config, data):
+		self.config = config
+		self.data = data
+		self.proxy = ConnectionManager(nodemanagerservice.Client, int(self.config.get('ClusterManager', 'nodeManagerPort')))
+		self.convertExceptions = boolean(config.get('ClusterManagerService', 'convertExceptions'))
+		self.log = logging.getLogger(__name__)
+		self.messageHandler = TashiLogHandler(config)
+		self.log.addHandler(self.messageHandler)
+		self.lastContacted = {}
+		self.decayedHosts = {}
+		self.decayedInstances = {}
+		self.expireHostTime = float(self.config.get('ClusterManagerService', 'expireHostTime'))
+		self.allowDecayed = float(self.config.get('ClusterManagerService', 'allowDecayed'))
+		now = time.time()
+		for instance in self.data.getInstances().itervalues():
+			instanceId = instance.id
+			instance = self.data.acquireInstance(instanceId)
+			instance.decayed = False
+			self.stateTransition(instance, None, InstanceState.Orphaned)
+			self.data.releaseInstance(instance)
+		for host in self.data.getHosts().itervalues():
+			hostId = host.id
+			host = self.data.acquireHost(hostId)
+			host.up = False
+			host.decayed = False
+			self.data.releaseHost(host)
+		self.decayLock = threading.Lock()
+		threading.Thread(target=self.monitorHosts).start()
+
+	def stateTransition(self, instance, old, cur):
+		if (old and instance.state != old):
+			self.data.releaseInstance(instance)
+			raise TashiException(d={'errno':Errors.IncorrectVmState,'msg':"VmState is not %s - it is %s" % (vmStates[old], vmStates[instance.state])})
+		instance.state = cur
+
+	def updateDecay(self, set, obj):
+		now = time.time()
+		self.decayLock.acquire()
+		if (obj.decayed and obj.id not in set):
+			set[obj.id] = now
+		elif (not obj.decayed and obj.id in set):
+			del set[obj.id]
+		self.decayLock.release()
+		
+
+	def monitorHosts(self):
+		# XXX: retry multiple hosts (iterate through them even with an exception)
+		while True:
+			now = time.time()
+			sleepFor = min(self.expireHostTime, self.allowDecayed)
+			try:
+				for k in self.lastContacted.keys():
+					if (self.lastContacted[k] < (now-self.expireHostTime)):
+						host = self.data.acquireHost(k)
+						try: 
+							self.log.warning('Host %s has expired after %f seconds' % (host.name, now-self.expireHostTime))
+							for instanceId in [instance.id for instance in self.data.getInstances().itervalues() if instance.hostId == host.id]:
+								instance = self.data.acquireInstance(instanceId)
+								instance.decayed = True
+								self.stateTransition(instance, None, InstanceState.Orphaned)
+								self.data.releaseInstance(instance)
+							host.up = False
+							host.decayed = False
+						finally:
+							self.data.releaseHost(host)
+						del self.lastContacted[k]
+					else:
+						sleepFor = min(self.lastContacted[k] + self.expireHostTime - now, sleepFor)
+				for hostId in self.decayedHosts.keys():
+					if (self.decayedHosts[hostId] < (now-self.allowDecayed)):
+						host = self.data.getHost(hostId)
+						self.log.warning('Fetching state from host %s because it is decayed' % (host.name))
+						hostProxy = self.proxy[host.name]
+						oldInstances = [i for i in self.data.getInstances().values() if i.hostId == host.id]
+						instances = [hostProxy.getVmInfo(vmId) for vmId in hostProxy.listVms()]
+						instanceIds = [i.id for i in instances]
+						for instance in instances:
+							if (instance.id not in self.data.getInstances()):
+								instance.hostId = host.id
+								instance = self.data.registerInstance(instance)
+								self.data.releaseInstance(instance)
+						for instance in oldInstances:
+							if (instance.id not in instanceIds):
+								instance = self.data.acquireInstance(instance.id)
+								self.data.removeInstance(instance)
+						self.decayedHosts[hostId] = now
+					else:
+						sleepFor = min(self.decayedHosts[hostId] + self.allowDecayed - now, sleepFor)
+				for instanceId in self.decayedInstances.keys():
+					try:
+						if (self.decayedInstances[instanceId] < (now-self.allowDecayed)):
+							self.log.warning('Fetching state on instance %d because it is decayed' % (instanceId))
+							try:
+								instance = self.data.getInstance(instanceId)
+							except TashiException, e:
+								if (e.errno == Errors.NoSuchInstanceId):
+									del self.decayedInstances[instanceId]
+									continue
+								else:
+									raise
+							host = self.data.getHost(instance.hostId)
+							hostProxy = self.proxy[host.name]
+							instance = hostProxy.getVmInfo(instance.vmId)
+							oldInstance = self.data.acquireInstance(instanceId)
+							oldInstance.state = instance.state
+							self.data.releaseInstance(oldInstance)
+							self.decayedInstances[instanceId] = now
+						else:
+							sleepFor = min(self.decayedInstances[instanceId] + self.allowDecayed - now, sleepFor)
+					except Exception, e:
+						self.log.exception('Exception in monitorHosts trying to get instance information')
+			except Exception, e:
+				self.log.exception('Exception in monitorHosts')
+			time.sleep(sleepFor)
+	
+	@RPC
+	def createVm(self, instance):
+		"""Function to add a VM to the list of pending VMs"""
+		instance.state = InstanceState.Pending
+		# XXX: Synchronize on MachineType
+		instance.typeObj = self.data.getMachineTypes()[instance.type]
+		instance.decayed = False
+		instance = self.data.registerInstance(instance)
+		self.data.releaseInstance(instance)
+		return instance
+	
+	@RPC
+	def shutdownVm(self, instanceId):
+		instance = self.data.acquireInstance(instanceId)
+		self.stateTransition(instance, InstanceState.Running, InstanceState.ShuttingDown)
+		self.data.releaseInstance(instance)
+		hostname = self.data.getHost(instance.hostId).name
+		try:
+			self.proxy[hostname].shutdownVm(instance.vmId)
+		except Exception:
+			self.log.exception('shutdownVm failed for host %s vmId %d' % (instance.hostname, instance.vmId))
+			raise
+		return
+	
+	@RPC
+	def destroyVm(self, instanceId):
+		instance = self.data.acquireInstance(instanceId)
+		if (instance.state is InstanceState.Pending or instance.state is InstanceState.Held):
+			self.data.removeInstance(instance)
+		elif (instance.state is InstanceState.Activating):
+			self.stateTransition(instance, InstanceState.Activating, InstanceState.Destroying)
+			self.data.releaseInstance(instance)
+		else:
+			self.stateTransition(instance, None, InstanceState.Destroying)
+			self.data.releaseInstance(instance)
+			hostname = self.data.getHost(instance.hostId).name
+			try:
+				self.proxy[hostname].destroyVm(instance.vmId)
+			except Exception:
+				self.log.exception('destroyVm failed for host %s vmId %d' % (hostname, instance.vmId))
+				raise
+		return
+	
+	@RPC
+	def suspendVm(self, instanceId, destination):
+		instance = self.data.acquireInstance(instanceId)
+		self.stateTransition(instance, InstanceState.Running, InstanceState.Suspending)
+		self.data.releaseInstance(instance)
+		suspendCookie = ""
+		hostname = self.data.getHost(instance.hostId).name
+		try:
+			self.proxy[hostname].suspendVm(instance.vmId, destination, suspendCookie)
+		except Exception:
+			self.log.exception('suspendVm failed for host %s vmId %d' % (hostname, instance.vmId))
+			raise
+		return
+	
+	@RPC
+	def resumeVm(self, instance, source):
+		instance.state = InstanceState.Pending
+		# XXX: Synchronize on MachineType
+		instance.typeObj = self.data.getMachineTypes()[instance.type]
+		instance.decayed = False
+		instance.hints['__resume_source'] = source
+		instance = self.data.registerInstance(instance)
+		self.data.releaseInstance(instance)
+		return instance
+	
+	@RPC
+	def migrateVm(self, instanceId, targetHostId):
+		instance = self.data.acquireInstance(instanceId)
+		try:
+			# FIXME: should these be acquire/release host?
+			targetHost = self.data.getHost(targetHostId)
+			sourceHost = self.data.getHost(instance.hostId)
+			# FIXME: Are these the correct state transitions?
+		except:
+			self.data.releaseInstance(instance)
+			raise
+		self.stateTransition(instance, InstanceState.Running, InstanceState.MigratePrep)
+		self.data.releaseInstance(instance)
+		try:
+			# Prepare the target
+			cookie = self.proxy[targetHost.name].prepReceiveVm(instance, sourceHost)
+		except Exception, e:
+			self.log.exception('prepReceiveVm failed')
+			raise
+		instance = self.data.acquireInstance(instance.id)
+		self.stateTransition(instance, InstanceState.MigratePrep, InstanceState.MigrateTrans)
+		self.data.releaseInstance(instance)
+		try:
+			# Send the VM
+			self.proxy[sourceHost.name].migrateVm(instance.vmId, targetHost, cookie)
+		except Exception, e:
+			self.log.exception('migrateVm failed')
+			raise
+		#instance = self.data.acquireInstance(instance.id)
+		#try:
+		#	instance.hostId = targetHost.id
+		#finally:
+		#	self.data.releaseInstance(instance)
+		try:
+			# Notify the target
+			vmId = self.proxy[targetHost.name].receiveVm(instance, cookie)
+		except Exception, e:
+			self.log.exception('receiveVm failed')
+			raise
+		#print 'VM %i Migrated!  New vmId=%i, new hostId=%i' % (instance.id, vmId, targetHostId)
+		return
+	
+	@RPC
+	def pauseVm(self, instanceId):
+		instance = self.data.acquireInstance(instanceId)
+		self.stateTransition(instance, InstanceState.Running, InstanceState.Pausing)
+		self.data.releaseInstance(instance)
+		hostname = self.data.getHost(instance.hostId).name
+		try:
+			self.proxy[hostname].pauseVm(instance.vmId)
+		except Exception:
+			self.log.exception('pauseVm failed on host %s with vmId %d' % (hostname, instance.vmId))
+			raise
+		instance = self.data.acquireInstance(instanceId)
+		self.stateTransition(instance, InstanceState.Pausing, InstanceState.Paused)
+		self.data.releaseInstance(instance)
+		return
+
+	@RPC
+	def unpauseVm(self, instanceId):
+		instance = self.data.acquireInstance(instanceId)
+		self.stateTransition(instance, InstanceState.Paused, InstanceState.Unpausing)
+		self.data.releaseInstance(instance)
+		hostname = self.data.getHost(instance.hostId).name
+		try:
+			self.proxy[hostname].unpauseVm(instance.vmId)
+		except Exception:
+			self.log.exception('unpauseVm failed on host %s with vmId %d' % (hostname, instance.vmId))
+			raise
+		instance = self.data.acquireInstance(instanceId)
+		self.stateTransition(instance, InstanceState.Unpausing, InstanceState.Running)
+		self.data.releaseInstance(instance)
+		return
+	
+	@RPC
+	def getMachineTypes(self):
+		return self.data.getMachineTypes().values()
+	
+	@RPC
+	def getHosts(self):
+		return self.data.getHosts().values()
+	
+	@RPC
+	def getNetworks(self):
+		return self.data.getNetworks().values()
+	
+	@RPC
+	def getUsers(self):
+		return self.data.getUsers().values()
+	
+	@RPC
+	def getInstances(self):
+		instances = self.data.getInstances().values()
+		for instance in instances:
+			if (instance.hostId):
+				instance.hostObj = self.data.getHost(instance.hostId)
+			else:
+				instance.hostObj = None
+			if (instance.userId):
+				instance.userObj = self.data.getUser(instance.userId)
+			else:
+				instance.userObj = None
+		return instances
+	
+#	@timed
+	@RPC
+	def registerNodeManager(self, host, instances):
+		"""Called by the NM every so often as a keep-alive/state polling -- state changes here are NOT AUTHORITATIVE"""
+		if (host.id == None):
+			hostList = [h for h in self.data.getHosts().itervalues() if h.name == host.name]
+			if (len(hostList) != 1):
+				raise TashiException(d={'errno':Errors.NoSuchHost, 'msg':'A host with name %s is not identifiable' % (host.name)})
+			host.id = hostList[0].id
+		oldHost = self.data.acquireHost(host.id)
+		if (oldHost.name != host.name):
+			self.data.releaseHost(oldHost)
+			raise TashiException(d={'errno':Errors.NoSuchHostId, 'msg':'Host id and hostname mismatch'})
+		try:
+			self.lastContacted[host.id] = time.time()
+			oldHost.memory = host.memory
+			oldHost.cores = host.cores
+			oldHost.up = True
+			oldHost.decayed = False
+			for instance in instances:
+				try:
+					oldInstance = self.data.acquireInstance(instance.id)
+				except TashiException, e:
+					if (e.errno == Errors.NoSuchInstanceId):
+						self.log.info('Host %s reported an instance %d that did not previously exist (decay)' % (host.name, instance.id))
+						oldHost.decayed = True
+						continue
+						#oldInstance = self.data.registerInstance(instance)
+					else:
+						raise
+				try:
+					if (oldInstance.hostId != host.id):
+						self.log.info('Host %s is claiming instance %d actually owned by hostId %s (decay)' % (host.name, oldInstance.id, str(oldInstance.hostId)))
+						oldHost.decayed = True
+						continue
+					oldInstance.decayed = (oldInstance.state != instance.state)
+					self.updateDecay(self.decayedInstances, oldInstance)
+					if (oldInstance.decayed):
+						self.log.info('State reported as %s instead of %s for instance %d on host %s (decay)' % (vmStates[instance.state], vmStates[oldInstance.state], instance.id, host.name))
+				finally:
+					self.data.releaseInstance(oldInstance)
+			instanceIds = [instance.id for instance in instances]
+			for instanceId in [instance.id for instance in self.data.getInstances().itervalues() if instance.hostId == host.id]:
+				if (instanceId not in instanceIds):
+					self.log.info('instance %d was not reported by host %s as expected (decay)' % (instanceId, host.name))
+					instance = self.data.acquireInstance(instanceId)
+					instance.decayed = True
+					self.updateDecay(self.decayedInstances, instance)
+					oldHost.decayed = True
+					self.data.releaseInstance(instance)
+		except Exception, e:
+			oldHost.decayed = True
+			raise
+		finally:
+			self.updateDecay(self.decayedHosts, oldHost)
+			self.data.releaseHost(oldHost)
+		return host.id
+	
+	@RPC
+	def vmUpdate(self, instanceId, instance, oldState):
+		try:
+			oldInstance = self.data.acquireInstance(instanceId)
+		except TashiException, e:
+			if (e.errno == Errors.NoSuchInstanceId):
+				self.log.exception('Got vmUpdate for unknown instanceId %d' % (instanceId))
+				return
+			else:
+				raise
+		if (instance.state == InstanceState.Exited):
+			oldInstance.decayed = False
+			self.updateDecay(self.decayedInstances, oldInstance)
+			self.data.removeInstance(oldInstance)
+			hostname = self.data.getHost(oldInstance.hostId).name
+			if (oldInstance.state not in [InstanceState.ShuttingDown, InstanceState.Destroying, InstanceState.Suspending]):
+				self.log.warning('Unexpected exit on %s of instance %d (vmId %d)' % (hostname, instanceId, oldInstance.vmId))
+		else:
+			if (instance.state):
+				if (oldState and oldInstance.state != oldState):
+					self.log.warning('Got vmUpdate of state from %s to %s, but the instance was previously %s' % (vmStates[oldState], vmStates[instance.state], vmStates[oldInstance.state]))
+				oldInstance.state = instance.state
+			if (instance.vmId):
+				oldInstance.vmId = instance.vmId
+			if (instance.hostId):
+				oldInstance.hostId = instance.hostId
+			oldInstance.decayed = False
+			self.updateDecay(self.decayedInstances, oldInstance)
+			self.data.releaseInstance(oldInstance)
+		return
+	
+	@RPC
+	def activateVm(self, instanceId, host):
+		dataHost = self.data.acquireHost(host.id)
+		if (dataHost.name != host.name):
+			self.data.releaseHost(dataHost)
+			raise TashiException(d={'errno':Errors.HostNameMismatch,'msg':"Mismatched target host"})
+		if (not dataHost.up):
+			self.data.releaseHost(dataHost)
+			raise TashiException(d={'errno':Errors.HostNotUp,'msg':"Target host is not up"})
+		if (dataHost.state != HostState.Normal):
+			self.data.releaseHost(dataHost)
+			raise TashiException(d={'errno':Errors.HostStateError,'msg':"Target host state is not normal"})
+		self.data.releaseHost(dataHost)
+		instance = self.data.acquireInstance(instanceId)
+		self.stateTransition(instance, InstanceState.Pending, InstanceState.Activating)
+		instance.hostId = host.id
+		self.data.releaseInstance(instance)
+		try:
+			if ('__resume_source' in instance.hints):
+				resumeVmId = self.proxy[host.name].resumeVm(instance, instance.hints['__resume_source'])
+				vmId = resumeVmId.vmId
+				suspendCookie = resumeVmId.suspendCookie
+			else:
+				vmId = self.proxy[host.name].instantiateVm(instance)
+		except Exception, e:
+			instance = self.data.acquireInstance(instanceId)
+			if (instance.state is InstanceState.Destroying): # Special case for if destroyVm is called during initialization and initialization fails
+				self.data.removeInstance(instance)
+			else:
+				self.stateTransition(instance, None, InstanceState.Held)
+				instance.hostId = None
+				self.data.releaseInstance(instance)
+			raise
+		instance = self.data.acquireInstance(instanceId)
+		instance.vmId = vmId
+		if (instance.state is InstanceState.Destroying): # Special case for if destroyVm is called during initialization
+			self.data.releaseInstnace(instance)
+			try:
+				self.proxy[host.name].destroyVm(vmId)
+			except Exception:
+				self.log.exception('destroyVm failed for host %s vmId %d' % (host.name, instance.vmId))
+				raise
+		else:
+			self.stateTransition(instance, InstanceState.Activating, InstanceState.Running)
+			self.data.releaseInstance(instance)
+		return

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/__init__.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/__init__.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/__init__.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/__init__.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,3 @@
+from datainterface import DataInterface
+from fromconfig import FromConfig
+from pickled import Pickled

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/datainterface.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/datainterface.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/datainterface.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/datainterface.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,54 @@
+class DataInterface(object):
+	"""Interface for a functional data access mechanism"""
+	def __init__(self, config):
+		if (self.__class__ is DataInterface):
+			raise NotImplementedError
+		self.config = config
+	
+	def registerInstance(self, instance):
+		raise NotImplementedError
+	
+	def acquireInstance(self, instanceId):
+		raise NotImplementedError
+	
+	def releaseInstance(self, instance):
+		raise NotImplementedError
+	
+	def removeInstance(self, instance):
+		raise NotImplementedError
+	
+	def acquireHost(self, hostId):
+		raise NotImplementedError
+	
+	def releaseHost(self, host):
+		raise NotImplementedError
+	
+	def getHosts(self):
+		raise NotImplementedError
+	
+	def getHost(self, id):
+		raise NotImplementedError
+	
+	def getInstances(self):
+		raise NotImplementedError
+	
+	def getInstance(self, id):
+		raise NotImplementedError
+	
+	def getMachineTypes(self):
+		raise NotImplementedError
+	
+	def getMachineType(self, id):
+		raise NotImplementedError
+	
+	def getNetworks(self):
+		raise NotImplementedError
+	
+	def getNetwork(self, id):
+		raise NotImplementedError
+	
+	def getUsers(self):
+		raise NotImplementedError
+	
+	def getUser(self, id):
+		raise NotImplementedError

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/fromconfig.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/fromconfig.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/fromconfig.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/fromconfig.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,153 @@
+import threading
+
+from tashi.services.ttypes import *
+from tashi.clustermanager.data import DataInterface
+
+class FromConfig(DataInterface):
+	def __init__(self, config):
+		DataInterface.__init__(self, config)
+		self.hosts = {}
+		self.instances = {}
+		self.machineTypes = {}
+		self.networks = {}
+		self.users = {}
+#		self.locks = {}
+		self.lockNames = {}
+		self.instanceLock = threading.Lock()
+		self.lockNames[self.instanceLock] = "instanceLock"
+		self.instanceIdLock = threading.Lock()
+		self.lockNames[self.instanceIdLock] = "instanceIdLock"
+		self.maxInstanceId = 1
+		for (name, value) in self.config.items("FromConfig"):
+			name = name.lower()
+			if (name.startswith("host")):
+				host = eval(value)
+				if (host.__class__ is not Host):
+					raise ValueError, "Entry %s is not a Host" % (name)
+				host._lock = threading.Lock()
+				self.lockNames[host._lock] = "h%d" % (host.id)
+				self.hosts[host.id] = host
+			if (name.startswith("machinetype")):
+				machineType = eval(value)
+				if (machineType.__class__ is not MachineType):
+					raise ValueError, "Entry %s is not a MachineType" % (name)
+				self.machineTypes[machineType.id] = machineType
+			if (name.startswith("network")):
+				network = eval(value)
+				if (network.__class__ is not Network):
+					raise ValueError, "Entry %s is not a Network" % (name)
+				self.networks[network.id] = network
+			if (name.startswith("user")):
+				user = eval(value)
+				if (user.__class__ is not User):
+					raise ValueError, "Entry %s is not a User" % (name)
+				self.users[user.id] = user
+	
+	def acquireLock(self, l):
+		l.acquire()
+#		self.locks[l] = threading.currentThread()
+	
+	def releaseLock(self, l):
+#		del self.locks[l]
+		l.release()
+	
+	def getNewInstanceId(self):
+		self.acquireLock(self.instanceIdLock)
+		instanceId = self.maxInstanceId
+		self.maxInstanceId = self.maxInstanceId + 1
+		self.releaseLock(self.instanceIdLock)
+		return instanceId
+	
+	def registerInstance(self, instance):
+		self.acquireLock(self.instanceLock)
+		try:
+			if (instance.id is not None and instance.id not in self.instances):
+				self.acquireLock(self.instanceIdLock)
+				if (instance.id >= self.maxInstanceId):
+					self.maxInstanceId = instance.id + 1
+				self.releaseLock(self.instanceIdLock)
+			else:
+				instance.id = self.getNewInstanceId()
+			instance._lock = threading.Lock()
+			self.lockNames[instance._lock] = "i%d" % (instance.id)
+			self.acquireLock(instance._lock)
+			self.instances[instance.id] = instance
+		finally:
+			self.releaseLock(self.instanceLock)
+		return instance
+	
+	def acquireInstance(self, instanceId):
+		self.acquireLock(self.instanceLock)
+		try:
+			instance = self.instances.get(instanceId, None)
+			if (instance is None):
+				raise TashiException(d={'errno':Errors.NoSuchInstanceId,'msg':"No such instanceId - %d" % (instanceId)})
+			self.acquireLock(instance._lock)
+		finally:
+			self.releaseLock(self.instanceLock)
+		return instance
+	
+	def releaseInstance(self, instance):
+		try:
+			if (instance.id not in self.instances): # MPR: should never be true, but good to check
+				raise TashiException(d={'errno':Errors.NoSuchInstanceId,'msg':"No such instanceId - %d" % (instanceId)})
+		finally:
+			self.releaseLock(instance._lock)
+	
+	def removeInstance(self, instance):
+		self.acquireLock(self.instanceLock)
+		try:
+			del self.instances[instance.id]
+			self.releaseLock(instance._lock)
+		finally:
+			self.releaseLock(self.instanceLock)
+	
+	def acquireHost(self, hostId):
+		host = self.hosts.get(hostId, None)
+		if (host is None):
+			raise TashiException(d={'errno':Errors.NoSuchHostId,'msg':"No such hostId - %s" % (hostId)})
+		self.acquireLock(host._lock)
+		return host
+	
+	def releaseHost(self, host):
+		try:
+			if (host.id not in self.hosts): # MPR: should never be true, but good to check
+				raise TashiException(d={'errno':Errors.NoSuchHostId,'msg':"No such hostId - %s" % (hostId)})
+		finally:
+			self.releaseLock(host._lock)
+	
+	def getHosts(self):
+		return self.hosts
+	
+	def getHost(self, id):
+		host = self.hosts.get(id, None)
+		if (not host):
+			raise TashiException(d={'errno':Errors.NoSuchHostId,'msg':"No such hostId - %s" % (id)})
+		return host
+
+	def getInstances(self):
+		return self.instances
+	
+	def getInstance(self, id):
+		instance = self.instances.get(id, None)
+		if (not instance):
+			raise TashiException(d={'errno':Errors.NoSuchInstanceId,'msg':"No such instanceId - %d" % (id)})
+		return instance
+	
+	def getMachineTypes(self):
+		return self.machineTypes
+	
+	def getMachineType(self, id):
+		return self.machineTypes[id]
+	
+	def getNetworks(self):
+		return self.networks
+	
+	def getNetwork(self, id):
+		return self.networks[id]
+	
+	def getUsers(self):
+		return self.users
+	
+	def getUser(self, id):
+		return self.users[id]

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/pickled.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/pickled.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/pickled.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/pickled.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,56 @@
+import cPickle
+import os
+import threading
+from tashi.services.ttypes import *
+from tashi.clustermanager.data import FromConfig, DataInterface
+
+class Pickled(FromConfig):
+	def __init__(self, config):
+		DataInterface.__init__(self, config)
+		self.file = self.config.get("Pickled", "file")
+		self.locks = {}
+		self.lockNames = {}
+		self.instanceLock = threading.Lock()
+		self.lockNames[self.instanceLock] = "instanceLock"
+		self.instanceIdLock = threading.Lock()
+		self.lockNames[self.instanceIdLock] = "instanceIdLock"
+		self.maxInstanceId = 1
+		self.load()
+	
+	def cleanInstances(self):
+		ci = {}
+		for i in self.instances.itervalues():
+			i2 = Instance(d=i.__dict__)
+			ci[i2.id] = i2
+		return ci
+	
+	def cleanHosts(self):
+		ch = {}
+		for h in self.hosts.itervalues():
+			h2 = Host(d=h.__dict__)
+			ch[h2.id] = h2
+		return ch
+	
+	def save(self):
+		file = open(self.file, "w")
+		cPickle.dump((self.cleanHosts(), self.cleanInstances(), self.machineTypes, self.networks, self.users), file)
+		file.close()
+
+	def load(self):
+		if (os.access(self.file, os.F_OK)):
+			file = open(self.file, "r")
+			(hosts, instances, machineTypes, networks, users) = cPickle.load(file)
+			file.close()
+		else:
+			(hosts, instances, machineTypes, networks, users) = ({}, {}, {}, {}, {})
+		self.hosts = hosts
+		self.instances = instances
+		self.machineTypes = machineTypes
+		self.networks = networks
+		self.users = users
+		for i in self.instances.itervalues():
+			i._lock = threading.Lock()
+			self.lockNames[i._lock] = "i%d" % (i.id)
+		for h in self.hosts.itervalues():
+			h._lock = threading.Lock()
+			self.lockNames[h._lock] = "h%d" % (h.id)

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/sql.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/sql.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/sql.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/clustermanager/data/sql.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,12 @@
+import time
+import types
+from tashi.clustermanager.data.datainterface import DataInterface
+
+class SQL(DataInterface):
+	def __init__(self, config):
+		DataInterface.__init__(self, config)
+		self.uri = self.config.get("SQL", "uri")
+		if (self.uri.startswith("sqlite://")):
+			raise NotImplementedError		
+		else:
+			raise ValueException, "Unknown SQL uri: %s" % (self.uri)

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/connectionmanager.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/connectionmanager.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/connectionmanager.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/connectionmanager.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,46 @@
+from thrift.transport.TSocket import TSocket, socket
+from thrift.protocol.TBinaryProtocol import TBinaryProtocol
+from thrift.transport.TTransport import TBufferedTransport
+
+class ConnectionManager(object):
+	def __init__(self, clientClass, port, timeout=10000.0):
+		self.clientClass = clientClass
+		self.timeout = timeout
+		self.port = port
+	
+	class anonClass(object):
+		def __init__(self, clientObject):
+			self.co = clientObject
+		
+		def __getattr__(self, name):
+			if (name.startswith("_")):
+				return self.__dict__[name]
+			def connectWrap(*args, **kw):
+				if (not self.co._iprot.trans.isOpen()):
+					self.co._iprot.trans.open()
+				try:
+					res = getattr(self.co, name)(*args, **kw)
+				except socket.error, e:
+					# Force a close for the case of a "Broken pipe"
+#					print "Forced a socket close"
+					self.co._iprot.trans.close()
+					self.co._iprot.trans.open()
+					res = getattr(self.co, name)(*args, **kw)
+					self.co._iprot.trans.close()
+					raise
+				self.co._iprot.trans.close()
+				return res
+			return connectWrap
+	
+	def __getitem__(self, hostname):
+                port = self.port
+                if len(hostname) == 2:
+                        port = hostname[1]
+                        hostname = hostname[0]
+		socket = TSocket(hostname, port)
+		socket.setTimeout(self.timeout)
+		transport = TBufferedTransport(socket)
+		protocol = TBinaryProtocol(transport)
+		client = self.clientClass(protocol)
+		client.__transport__ = transport
+		return self.anonClass(client)

Added: incubator/tashi/import/tashi-intel-r399/src/tashi/dfs/__init__.py
URL: http://svn.apache.org/viewvc/incubator/tashi/import/tashi-intel-r399/src/tashi/dfs/__init__.py?rev=710072&view=auto
==============================================================================
--- incubator/tashi/import/tashi-intel-r399/src/tashi/dfs/__init__.py (added)
+++ incubator/tashi/import/tashi-intel-r399/src/tashi/dfs/__init__.py Mon Nov  3 06:45:25 2008
@@ -0,0 +1,2 @@
+from dfsinterface import DfsInterface
+from vfs import Vfs



Mime
View raw message