cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From t..@apache.org
Subject git commit: updated refs/heads/requestsformarvin to 20ce63f
Date Wed, 17 Apr 2013 11:36:53 GMT
Updated Branches:
  refs/heads/requestsformarvin [created] 20ce63fcf


WIP refactor for cloudstackConnection

Moving to use requests and refactoring to include post data

Signed-off-by: Prasanna Santhanam <tsp@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/cloudstack/commit/20ce63fc
Tree: http://git-wip-us.apache.org/repos/asf/cloudstack/tree/20ce63fc
Diff: http://git-wip-us.apache.org/repos/asf/cloudstack/diff/20ce63fc

Branch: refs/heads/requestsformarvin
Commit: 20ce63fcf053f82a9b090f9a33c9fdc9f2897f54
Parents: fad55c5
Author: Prasanna Santhanam <tsp@apache.org>
Authored: Wed Apr 17 17:06:15 2013 +0530
Committer: Prasanna Santhanam <tsp@apache.org>
Committed: Wed Apr 17 17:06:15 2013 +0530

----------------------------------------------------------------------
 tools/marvin/marvin/asyncJobMgr.py          |    2 +-
 tools/marvin/marvin/cloudstackConnection.py |  239 +++++++++++-----------
 tools/marvin/marvin/cloudstackTestClient.py |    3 +-
 tools/marvin/marvin/codegenerator.py        |    4 +-
 tools/marvin/marvin/deployDataCenter.py     |    1 -
 tools/marvin/marvin/jsonHelper.py           |    7 +-
 6 files changed, 130 insertions(+), 126 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/asyncJobMgr.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/asyncJobMgr.py b/tools/marvin/marvin/asyncJobMgr.py
index 935bebe..40304fa 100644
--- a/tools/marvin/marvin/asyncJobMgr.py
+++ b/tools/marvin/marvin/asyncJobMgr.py
@@ -54,7 +54,7 @@ class workThread(threading.Thread):
         
         try:
             self.lock.acquire()
-            result = self.connection.pollAsyncJob(job.jobId, job.responsecls).jobresult
+            result = self.connection.poll(job.jobId, job.responsecls).jobresult
         except cloudstackException.cloudstackAPIException, e:
             result = str(e)
         finally:

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/cloudstackConnection.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/cloudstackConnection.py b/tools/marvin/marvin/cloudstackConnection.py
index 1caeef3..06c846c 100644
--- a/tools/marvin/marvin/cloudstackConnection.py
+++ b/tools/marvin/marvin/cloudstackConnection.py
@@ -15,126 +15,133 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import urllib2
+import requests
 import urllib
-import httplib
 import base64
 import hmac
 import hashlib
-import json
-import xml.dom.minidom
-import types
 import time
-import inspect
 import cloudstackException
 from cloudstackAPI import * 
 import jsonHelper
+from requests import ConnectionError
+from requests import HTTPError
+from requests import Timeout
+from requests import RequestException
 
 class cloudConnection(object):
-    def __init__(self, mgtSvr, port=8096, apiKey = None, securityKey = None, asyncTimeout=3600,
logging=None, protocol='http', path='/client/api'):
+    def __init__(self, mgtSvr, port=8096, apiKey=None, securityKey=None, asyncTimeout=3600,
logging=None,
+                 scheme='http', path='client/api'):
         self.apiKey = apiKey
         self.securityKey = securityKey
         self.mgtSvr = mgtSvr
         self.port = port
         self.logging = logging
-        if protocol != 'http' and protocol != 'https':
-            raise ValueError("Protocol must be 'http' or 'https'.")
-        else:
-            self.protocol=protocol
         self.path = path
-        if port == 8096 or (self.apiKey == None and self.securityKey == None):
-            self.auth = False
-        else:
-            self.auth = True
         self.retries = 5
         self.asyncTimeout = asyncTimeout
-    
-    def close(self):
-        try:
-            self.connection.close()
-        except:
-            pass
-    
+        self.auth = True
+        if port == 8096 or \
+           (self.apiKey == None and self.securityKey == None):
+            self.auth = False
+        if scheme not in ['http', 'https']:
+                raise RequestException("Protocol must be HTTP")
+        self.protocol = scheme
+        self.baseurl = "%s://%s:%d/%s"%(self.protocol, self.mgtSvr, self.port, self.path)
+
     def __copy__(self):
-        return cloudConnection(self.mgtSvr, self.port, self.apiKey, self.securityKey, self.asyncTimeout,
self.logging, self.protocol, self.path)
-    
-    def make_request_with_auth(self, command, requests={}):
-        requests["command"] = command
-        requests["apiKey"] = self.apiKey
-        requests["response"] = "json"
-        request = zip(requests.keys(), requests.values())
-        request.sort(key=lambda x: str.lower(x[0]))
-        
-        requestUrl = "&".join(["=".join([r[0], urllib.quote_plus(str(r[1]))]) for r in
request])
-        hashStr = "&".join(["=".join([str.lower(r[0]), str.lower(urllib.quote_plus(str(r[1]))).replace("+",
"%20")]) for r in request])
-
-        sig = urllib.quote_plus(base64.encodestring(hmac.new(self.securityKey, hashStr, hashlib.sha1).digest()).strip())
-        requestUrl += "&signature=%s"%sig
+        return cloudConnection(self.mgtSvr, self.port, self.apiKey, self.securityKey, self.asyncTimeout,
+            self.logging, self.protocol, self.path)
 
-        try:
-            self.connection = urllib2.urlopen("%s://%s:%d%s?%s"%(self.protocol, self.mgtSvr,
self.port, self.path, requestUrl))
-            if self.logging is not None:
-                self.logging.debug("sending GET request: %s"%requestUrl)
-            response = self.connection.read()
-            if self.logging is not None:
-                self.logging.info("got response: %s"%response)
-        except IOError, e:
-            if hasattr(e, 'reason'):
-                if self.logging is not None:
-                    self.logging.critical("failed to reach %s because of %s"%(self.mgtSvr,
e.reason))
-            elif hasattr(e, 'code'):
-                if self.logging is not None:
-                    self.logging.critical("server returned %d error code"%e.code)
-            raise e
-        except httplib.HTTPException, h:
-            if self.logging is not None:
-                self.logging.debug("encountered http Exception %s"%h.args)
-            if self.retries > 0:
-                self.retries = self.retries - 1
-                self.make_request_with_auth(command, requests)
-            else:
-                self.retries = 5
-                raise h
-        else:
-            return response
-        
-    def make_request_without_auth(self, command, requests={}):
-        requests["command"] = command
-        requests["response"] = "json" 
-        requests = zip(requests.keys(), requests.values())
-        requestUrl = "&".join(["=".join([request[0], urllib.quote_plus(str(request[1]))])
for request in requests])
-
-        self.connection = urllib2.urlopen("%s://%s:%d%s?%s"%(self.protocol, self.mgtSvr,
self.port, self.path, requestUrl))
-        if self.logging is not None:
-            self.logging.debug("sending GET request without auth: %s"%requestUrl)
-        response = self.connection.read()
-        if self.logging is not None:
-            self.logging.info("got response: %s"%response)
-        return response
-    
-    def pollAsyncJob(self, jobId, response):
+    def poll(self, jobid, response):
+        """
+        polls the completion of a given jobid
+        @param jobid:
+        @param response:
+        @return:
+        """
         cmd = queryAsyncJobResult.queryAsyncJobResultCmd()
-        cmd.jobid = jobId
+        cmd.jobid = jobid
         timeout = self.asyncTimeout
-        
+
         while timeout > 0:
-            asyncResonse = self.make_request(cmd, response, True)
-            
+            asyncResonse = self.marvin_request(cmd, response, True)
+
             if asyncResonse.jobstatus == 2:
                 raise cloudstackException.cloudstackAPIException("asyncquery", asyncResonse.jobresult)
             elif asyncResonse.jobstatus == 1:
                 return asyncResonse
-            
+
             time.sleep(5)
             if self.logging is not None:
-                self.logging.debug("job: %s still processing, will timeout in %ds"%(jobId,
timeout))
+                self.logging.debug("job: %s still processing, will timeout in %ds"%(jobid,
timeout))
             timeout = timeout - 5
-            
-        raise cloudstackException.cloudstackAPIException("asyncquery", "Async job timeout
%s"%jobId)
-    
-    def make_request(self, cmd, response = None, raw=False):
-        commandName = cmd.__class__.__name__.replace("Cmd", "")
-        isAsync = "false"
+
+        raise cloudstackException.cloudstackAPIException("asyncquery", "Async job timeout
%s"%jobid)
+
+    def sign(self, payload):
+        """
+        signs a given request URL when the apiKey and secretKey are known
+
+        @param payload: dict of GET params to be signed
+        @return: the signature of the payload
+        """
+        params = zip(payload.keys(), map(lambda v: urllib.quote_plus(v), payload.values()))
+        params.sort(key=lambda k: str.lower(k[0]))
+        hashStr = "&".join(
+            ["=".join(
+                [str.lower(r[0]), str.lower(urllib.quote_plus(str(r[1]))).replace("+", "%20")]
+            ) for r in params]
+        )
+        signature = urllib.quote_plus(
+            base64.encodestring(hmac.new(self.securityKey, hashStr, hashlib.sha1).digest()).strip()
+        )
+        return signature
+
+    def request(self, command, auth=True, payload={}, data={}):
+        """
+        Makes requests on the `integration.api.port`
+        @param command: cloudstack API command name eg: deployVirtualMachineCommand
+        @param auth: Authentication (apikey,secretKey) => True, else False
+        @param payload: GET param data composed as a dictionary of key,value pairs
+        @param data: POST data as a dictionary
+        @return:
+        """
+        payload["command"] = command
+        payload["response"] = "json"
+
+        if auth:
+            payload["apiKey"] = self.apiKey
+            signature = self.sign(payload)
+            payload["signature"] = signature
+
+
+        try:
+            if data:
+                response = requests.get(self.baseurl, params=payload, data=data)
+            else:
+                response = requests.get(self.baseurl, params=payload)
+        except ConnectionError, c:
+            self.logging.debug("Connection refused. Reason: %s"%(self.baseurl, c))
+            raise c
+        except HTTPError, h:
+            self.logging.debug("Server returned error code: %s"%h)
+            raise h
+        except Timeout, t:
+            self.logging.debug("Connection timed out with %s"%t)
+            raise t
+        except RequestException,r:
+            self.logging.debug("Error returned by server %s"%r)
+            raise r
+        else:
+            return response
+
+    def sanitize_command(self, cmd):
+        """
+        Removes None values, Validates all required params are present
+        @param cmd: Cmd object eg: createPhysicalNetwork
+        @return:
+        """
         requests = {}
         required = []
         for attribute in dir(cmd):
@@ -145,11 +152,11 @@ class cloudConnection(object):
                     required = getattr(cmd, attribute)
                 else:
                     requests[attribute] = getattr(cmd, attribute)
-        
+
+        cmdname = cmd.__class__.__name__.replace("Cmd", "")
         for requiredPara in required:
             if requests[requiredPara] is None:
-                raise cloudstackException.cloudstackAPIException(commandName, "%s is required"%requiredPara)
-        '''remove none value'''
+                raise cloudstackException.cloudstackAPIException(cmdname, "%s is required"%requiredPara)
         for param, value in requests.items():
             if value is None:
                 requests.pop(param)
@@ -166,28 +173,30 @@ class cloudConnection(object):
                             for k,v in val.iteritems():
                                 requests["%s[%d].%s"%(param,i,k)] = v
                             i = i + 1
-        
-        if self.logging is not None:
-            self.logging.info("sending command: %s %s"%(commandName, str(requests)))
-        result = None
+        return cmdname, isAsync, requests
+
+    def marvin_request(self, cmd, data={}, response_type=None):
+        """
+        Requester for marvin command objects
+        @param cmd: marvin's command from cloudstackAPI
+        @param data: any data to be sent in as POST
+        @param response_type: response type of the command in cmd
+        @param raw:
+        @return:
+        """
+        cmdname, isAsync, payload = self.sanitize_command(cmd)
+        self.logging.info("sending command: %s %s"%(cmdname, str(payload)))
         if self.auth:
-            result = self.make_request_with_auth(commandName, requests)
+            response = self.request(cmdname, auth=True, payload=payload, data=data)
         else:
-            result = self.make_request_without_auth(commandName, requests)
-        
-        if result is None:
-            return None
-        
-        result = jsonHelper.getResultObj(result, response)
-        if raw or isAsync == "false":
-            return result
+            response = self.request(cmdname, auth=False, payload=payload, data=data)
+
+        self.logging.info("Request: %s Response: %s"%(response.url, response.text))
+        response = jsonHelper.getResultObj(response.json, response_type)
+
+        if isAsync == "false":
+            return response
         else:
-            asynJobId = result.jobid
-            result = self.pollAsyncJob(asynJobId, response)
-            return result.jobresult
-        
-if __name__ == '__main__':
-    xml = '<?xml version="1.0" encoding="ISO-8859-1"?><deployVirtualMachineResponse><virtualmachine><id>407</id><name>i-1-407-RS3</name><displayname>i-1-407-RS3</displayname><account>system</account><domainid>1</domainid><domain>ROOT</domain><created>2011-07-30T14:45:19-0700</created><state>Running</state><haenable>false</haenable><zoneid>1</zoneid><zonename>CA1</zonename><hostid>3</hostid><hostname>kvm-50-205</hostname><templateid>4</templateid><templatename>CentOS
5.5(64-bit) no GUI (KVM)</templatename><templatedisplaytext>CentOS 5.5(64-bit)
no GUI (KVM)</templatedisplaytext><passwordenabled>false</passwordenabled><serviceofferingid>1</serviceofferingid><serviceofferingname>Small
Instance</serviceofferingname><cpunumber>1</cpunumber><cpuspeed>500</cpuspeed><memory>512</memory><guestosid>112</guestosid><rootdeviceid>0</rootdeviceid><rootdevicetype>NetworkFilesystem</rootdevicetype><nic><id>380</id><networkid>203</networkid><netmask>255.255.255.0</netmask><gateway>65.19.181.1</gatew
 ay><ipaddress>65.19.181.110</ipaddress><isolationuri>vlan://65</isolationuri><broadcasturi>vlan://65</broadcasturi><traffictype>Guest</traffictype><type>Direct</type><isdefault>true</isdefault><macaddress>06:52:da:00:00:08</macaddress></nic><hypervisor>KVM</hypervisor></virtualmachine></deployVirtualMachineResponse>'
-    conn = cloudConnection(None)
-    
-    print conn.paraseReturnXML(xml, deployVirtualMachine.deployVirtualMachineResponse())
+            asyncJobId = response.jobid
+            response = self.poll(asyncJobId, response_type)
+            return response.jobresult

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/cloudstackTestClient.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/cloudstackTestClient.py b/tools/marvin/marvin/cloudstackTestClient.py
index e4735e4..85552ed 100644
--- a/tools/marvin/marvin/cloudstackTestClient.py
+++ b/tools/marvin/marvin/cloudstackTestClient.py
@@ -111,7 +111,8 @@ class cloudstackTestClient(object):
             apiKey = registerUserRes.apikey
             securityKey = registerUserRes.secretkey
         
-        newUserConnection = cloudstackConnection.cloudConnection(self.connection.mgtSvr,
self.connection.port, apiKey, securityKey, self.connection.asyncTimeout, self.connection.logging)
+        newUserConnection = cloudstackConnection.cloudConnection(self.connection.mgtSvr,
self.connection.port,
+            apiKey, securityKey, self.connection.asyncTimeout, self.connection.logging)
         self.userApiClient = cloudstackAPIClient.CloudStackAPIClient(newUserConnection)
         self.userApiClient.connection = newUserConnection
         return self.userApiClient

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/codegenerator.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/codegenerator.py b/tools/marvin/marvin/codegenerator.py
index 0622e5d..b3b2a86 100644
--- a/tools/marvin/marvin/codegenerator.py
+++ b/tools/marvin/marvin/codegenerator.py
@@ -184,9 +184,9 @@ class codeGenerator:
         body += "\n"
 
         for cmdName in self.cmdsName:
-            body += self.space + 'def %s(self,command):\n'%cmdName
+            body += self.space + 'def %s(self, command, postdata={}):\n'%cmdName
             body += self.space + self.space + 'response = %sResponse()\n'%cmdName
-            body += self.space + self.space + 'response = self.connection.make_request(command,
response)\n'
+            body += self.space + self.space + 'response = self.connection.marvin_request(command,
data=postdata, response_type=response)\n'
             body += self.space + self.space + 'return response\n'
             body += '\n'
 

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/deployDataCenter.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/deployDataCenter.py b/tools/marvin/marvin/deployDataCenter.py
index 2e270a7..d358789 100644
--- a/tools/marvin/marvin/deployDataCenter.py
+++ b/tools/marvin/marvin/deployDataCenter.py
@@ -407,7 +407,6 @@ class deployDataCenters():
                                             logging=self.testClientLogger)
         if mgt.apiKey is None:
             apiKey, securityKey = self.registerApiKey()
-            self.testClient.close()
             self.testClient = \
             cloudstackTestClient.cloudstackTestClient(mgt.mgtSvrIp, 8080, \
                                                       apiKey, securityKey, \

http://git-wip-us.apache.org/repos/asf/cloudstack/blob/20ce63fc/tools/marvin/marvin/jsonHelper.py
----------------------------------------------------------------------
diff --git a/tools/marvin/marvin/jsonHelper.py b/tools/marvin/marvin/jsonHelper.py
index 652cce0..37363bc 100644
--- a/tools/marvin/marvin/jsonHelper.py
+++ b/tools/marvin/marvin/jsonHelper.py
@@ -19,7 +19,6 @@ import cloudstackException
 import json
 import inspect
 from cloudstackAPI import *
-import pdb
 
 class jsonLoader:
     '''The recursive class for building and representing objects with.'''
@@ -113,12 +112,8 @@ def finalizeResultObj(result, responseName, responsecls):
             return result
     else:
         return result
-                    
-            
-        
+
 def getResultObj(returnObj, responsecls=None):
-    returnObj = json.loads(returnObj)
-    
     if len(returnObj) == 0:
         return None
     responseName = filter(lambda a: a!=u'cloudstack-version', returnObj.keys())[0]


Mime
View raw message