cloudstack-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From mc...@apache.org
Subject [16/51] [abbrv] git commit: refs/heads/vim51_win8 - CLOUDSTACK-1037: Fix cloudmonkey's caching, autocompletion and printing
Date Tue, 05 Feb 2013 18:06:57 GMT
CLOUDSTACK-1037: Fix cloudmonkey's caching, autocompletion and printing

Signed-off-by: Rohit Yadav <bhaisaab@apache.org>


Project: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/commit/b5a2e998
Tree: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/tree/b5a2e998
Diff: http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/diff/b5a2e998

Branch: refs/heads/vim51_win8
Commit: b5a2e998091eb15666d4ff634880f2c3ed994ea5
Parents: 5476391
Author: Rohit Yadav <bhaisaab@apache.org>
Authored: Fri Feb 1 22:01:07 2013 -0800
Committer: Rohit Yadav <bhaisaab@apache.org>
Committed: Sat Feb 2 08:39:45 2013 -0800

----------------------------------------------------------------------
 tools/cli/cloudmonkey/cachemaker.py  |   84 ++++++++++++++-------
 tools/cli/cloudmonkey/cloudmonkey.py |  112 ++++++++++++++++------------
 tools/cli/cloudmonkey/config.py      |    9 +--
 tools/cli/cloudmonkey/printer.py     |   17 +----
 tools/cli/pom.xml                    |   34 ---------
 5 files changed, 126 insertions(+), 130 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b5a2e998/tools/cli/cloudmonkey/cachemaker.py
----------------------------------------------------------------------
diff --git a/tools/cli/cloudmonkey/cachemaker.py b/tools/cli/cloudmonkey/cachemaker.py
index 264fa60..5886c62 100644
--- a/tools/cli/cloudmonkey/cachemaker.py
+++ b/tools/cli/cloudmonkey/cachemaker.py
@@ -19,9 +19,9 @@
 try:
     import json
     import os
-    import re
+    import types
 
-    from requester import monkeyrequest
+    from config import cache_file
 except ImportError, e:
     import sys
     print "ImportError", e
@@ -35,52 +35,79 @@ def getvalue(dictionary, key):
         return None
 
 
-def csv_str_as_list(string):
+def splitcsvstring(string):
     if string is not None:
         return filter(lambda x: x.strip() != '', string.split(','))
     else:
         return []
 
 
-def cachegen_from_file(json_file):
+def splitverbsubject(string):
+    idx = 0
+    for char in string:
+        if char.islower():
+            idx += 1
+        else:
+            break
+    return string[:idx].lower(), string[idx:].lower()
+
+
+def savecache(apicache, json_file):
+    """
+    Saves apicache dictionary as json_file, returns dictionary as indented str
+    """
+    apicachestr = json.dumps(apicache, indent=2)
+    with open(json_file, 'w') as cache_file:
+        cache_file.write(apicachestr)
+    return apicachestr
+
+
+def loadcache(json_file):
+    """
+    Loads json file as dictionary, feeds it to monkeycache and spits result
+    """
     f = open(json_file, 'r')
     data = f.read()
     f.close()
     try:
-        apis = json.loads(data)
+        apicache = json.loads(data)
     except ValueError, e:
-        print "Error processing json in cachegen()", e
-    return cachegen(apis)
+        print "Error processing json:", json_file, e
+        return {}
+    return apicache
 
 
-def cachegen(apis):
-    pattern = re.compile("[A-Z]")
+def monkeycache(apis):
+    """
+    Feed this a dictionary of api bananas, it spits out processed cache
+    """
+    if isinstance(type(apis), types.NoneType):
+        return {}
     responsekey = filter(lambda x: 'response' in x, apis.keys())
 
     if len(responsekey) == 0:
-        print "[cachegen] Invalid dictionary, has no response"
+        print "[monkeycache] Invalid dictionary, has no response"
         return None
     if len(responsekey) != 1:
-        print "[cachegen] Multiple responsekeys, chosing first one"
+        print "[monkeycache] Multiple responsekeys, chosing first one"
 
     responsekey = responsekey[0]
     verbs = set()
     cache = {}
     cache['count'] = getvalue(apis[responsekey], 'count')
+    cache['asyncapis'] = []
 
     for api in getvalue(apis[responsekey], 'api'):
         name = getvalue(api, 'name')
-        response = getvalue(api, 'response')
-
-        idx = pattern.search(name).start()
-        verb = name[:idx]
-        subject = name[idx:]
+        verb, subject = splitverbsubject(name)
 
         apidict = {}
         apidict['name'] = name
         apidict['description'] = getvalue(api, 'description')
         apidict['isasync'] = getvalue(api, 'isasync')
-        apidict['related'] = csv_str_as_list(getvalue(api, 'related'))
+        if apidict['isasync']:
+            cache['asyncapis'].append(name)
+        apidict['related'] = splitcsvstring(getvalue(api, 'related'))
 
         required = []
         apiparams = []
@@ -91,15 +118,16 @@ def cachegen(apis):
             apiparam['required'] = (getvalue(param, 'required') is True)
             apiparam['length'] = int(getvalue(param, 'length'))
             apiparam['type'] = getvalue(param, 'type')
-            apiparam['related'] = csv_str_as_list(getvalue(param, 'related'))
+            apiparam['related'] = splitcsvstring(getvalue(param, 'related'))
             if apiparam['required']:
                 required.append(apiparam['name'])
             apiparams.append(apiparam)
 
         apidict['requiredparams'] = required
         apidict['params'] = apiparams
-        apidict['response'] = getvalue(api, 'response')
-        cache[verb] = {subject: apidict}
+        if verb not in cache:
+            cache[verb] = {}
+        cache[verb][subject] = apidict
         verbs.add(verb)
 
     cache['verbs'] = list(verbs)
@@ -108,7 +136,7 @@ def cachegen(apis):
 
 def main(json_file):
     """
-    cachegen.py creates a precache datastore of all available apis of
+    cachemaker.py creates a precache datastore of all available apis of
     CloudStack and dumps the precache dictionary in an
     importable python module. This way we cheat on the runtime overhead of
     completing commands and help docs. This reduces the overall search and
@@ -116,7 +144,7 @@ def main(json_file):
     """
     f = open("precache.py", "w")
     f.write("""# -*- coding: utf-8 -*-
-# Auto-generated code by cachegen.py
+# Auto-generated code by cachemaker.py
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
 # distributed with this work for additional information
@@ -133,13 +161,13 @@ def main(json_file):
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.""")
-    f.write("\nprecache = %s" % cachegen_from_file(json_file))
+    f.write("\napicache = %s" % loadcache(json_file))
     f.close()
 
 if __name__ == "__main__":
-    json_file = 'listapis.json'
-    if os.path.exists(json_file):
-        main(json_file)
+    print "[cachemaker] Pre-caching using user's cloudmonkey cache", cache_file
+    if os.path.exists(cache_file):
+        main(cache_file)
     else:
-        pass
-        #print "[ERROR] cli:cachegen is unable to locate %s" % json_file
+        print "[cachemaker] Unable to cache apis, file not found", cache_file
+        print "[cachemaker] Run cloudmonkey sync to generate cache"

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b5a2e998/tools/cli/cloudmonkey/cloudmonkey.py
----------------------------------------------------------------------
diff --git a/tools/cli/cloudmonkey/cloudmonkey.py b/tools/cli/cloudmonkey/cloudmonkey.py
index 49f3ee9..6ebeb6a 100644
--- a/tools/cli/cloudmonkey/cloudmonkey.py
+++ b/tools/cli/cloudmonkey/cloudmonkey.py
@@ -24,28 +24,32 @@ try:
     import logging
     import os
     import pdb
-    import re
     import shlex
     import sys
+    import types
 
     from urllib2 import HTTPError, URLError
     from httplib import BadStatusLine
 
-    from config import __version__, config_file
-    from config import precached_verbs, read_config, write_config
+    from config import __version__, cache_file
+    from config import read_config, write_config
+
     from printer import monkeyprint
     from requester import monkeyrequest
+    from cachemaker import loadcache, savecache, monkeycache
+    from cachemaker import splitverbsubject
 
     from prettytable import PrettyTable
-    from marvin.cloudstackConnection import cloudConnection
-    from marvin.cloudstackException import cloudstackAPIException
-    from marvin.cloudstackAPI import *
-    from marvin import cloudstackAPI
 except ImportError, e:
     print "Import error in %s : %s" % (__name__, e)
     import sys
     sys.exit()
 
+try:
+    from precache import apicache
+except ImportError:
+    apicache = {}
+
 # Fix autocompletion issue, can be put in .pythonstartup
 try:
     import readline
@@ -60,23 +64,21 @@ else:
 
 log_fmt = '%(asctime)s - %(filename)s:%(lineno)s - [%(levelname)s] %(message)s'
 logger = logging.getLogger(__name__)
-completions = cloudstackAPI.__all__
 
 
 class CloudMonkeyShell(cmd.Cmd, object):
     intro = ("☁ Apache CloudStack 🐵 cloudmonkey " + __version__ +
              ". Type help or ? to list commands.\n")
     ruler = "="
-    apicache = {}
-    # datastructure {'verb': {cmd': ['api', [params], doc, required=[]]}}
-    cache_verbs = precached_verbs
+    cache_file = cache_file
+    ## datastructure {'verb': {cmd': ['api', [params], doc, required=[]]}}
+    #cache_verbs = apicache
     config_options = []
 
-    def __init__(self, pname, verbs):
+    def __init__(self, pname):
         self.program_name = pname
-        self.verbs = verbs
-
         self.config_options = read_config(self.get_attr, self.set_attr)
+        self.loadcache()
         self.prompt = self.prompt.strip() + " "  # Cosmetic fix for prompt
 
         logging.basicConfig(filename=self.log_file,
@@ -111,8 +113,27 @@ class CloudMonkeyShell(cmd.Cmd, object):
             except KeyboardInterrupt:
                 print("^C")
 
+    def loadcache(self):
+        if os.path.exists(self.cache_file):
+            self.apicache = loadcache(self.cache_file)
+        else:
+            self.apicache = apicache
+        self.verbs = apicache['verbs']
+
     def monkeyprint(self, *args):
-        monkeyprint((self.color == 'true'), *args)
+        output = ""
+        try:
+            for arg in args:
+                if isinstance(type(arg), types.NoneType):
+                    continue
+                output += str(arg)
+        except Exception, e:
+            print e
+
+        if self.color == 'true':
+            monkeyprint(output)
+        else:
+            print output
 
     def print_result(self, result, result_filter=None):
         if result is None or len(result) == 0:
@@ -186,6 +207,9 @@ class CloudMonkeyShell(cmd.Cmd, object):
         if self.pipe_runner(args):
             return
 
+        apiname = args.partition(' ')[0]
+        verb, subject = splitverbsubject(apiname)
+
         lexp = shlex.shlex(args.strip())
         lexp.whitespace = " "
         lexp.whitespace_split = True
@@ -196,7 +220,6 @@ class CloudMonkeyShell(cmd.Cmd, object):
             if next_val is None:
                 break
             args.append(next_val)
-        api_name = args[0]
 
         args_dict = dict(map(lambda x: [x.partition("=")[0],
                                         x.partition("=")[2]],
@@ -207,22 +230,15 @@ class CloudMonkeyShell(cmd.Cmd, object):
                                   map(lambda x: x.strip(),
                                       args_dict.pop('filter').split(',')))
 
-        for attribute in args_dict.keys():
-            setattr(api_cmd, attribute, args_dict[attribute])
-
-        #command = api_cmd()
-        #missing_args = filter(lambda x: x not in args_dict.keys(),
-        #                      command.required)
-
-        #if len(missing_args) > 0:
-        #    self.monkeyprint("Missing arguments: ", ' '.join(missing_args))
-        #    return
+        missing_args = filter(lambda x: x not in args_dict.keys(),
+                              self.apicache[verb][subject]['requiredparams'])
 
-        isAsync = False
-        #if "isAsync" in dir(command):
-        #    isAsync = (command.isAsync == "true")
+        if len(missing_args) > 0:
+            self.monkeyprint("Missing arguments: ", ' '.join(missing_args))
+            return
 
-        result = self.make_request(api_name, args_dict, isAsync)
+        result = self.make_request(apiname, args_dict,
+                                   apiname in self.apicache['asyncapis'])
         if result is None:
             return
         try:
@@ -248,17 +264,19 @@ class CloudMonkeyShell(cmd.Cmd, object):
         search_string = ""
 
         if separator != " ":   # Complete verb subjects
-            autocompletions = self.cache_verbs[verb].keys()
+            autocompletions = self.apicache[verb].keys()
             search_string = subject
         else:                  # Complete subject params
             autocompletions = map(lambda x: x + "=",
-                                  self.cache_verbs[verb][subject][1])
+                                  map(lambda x: x['name'],
+                                      self.apicache[verb][subject]['params']))
             search_string = text
 
         if self.tabularize == "true" and subject != "":
             autocompletions.append("filter=")
         return [s for s in autocompletions if s.startswith(search_string)]
 
+
     def do_sync(self, args):
         """
         Asks cloudmonkey to discovery and sync apis available on user specified
@@ -266,9 +284,9 @@ class CloudMonkeyShell(cmd.Cmd, object):
         it rollbacks last datastore or api precached datastore.
         """
         response = self.make_request("listApis")
-        f = open('test.json', "w")
-        f.write(json.dumps(response))
-        f.close()
+        self.apicache = monkeycache(response)
+        savecache(self.apicache, self.cache_file)
+        self.loadcache()
 
     def do_api(self, args):
         """
@@ -282,11 +300,6 @@ class CloudMonkeyShell(cmd.Cmd, object):
         else:
             self.monkeyprint("Please use a valid syntax")
 
-    def complete_api(self, text, line, begidx, endidx):
-        mline = line.partition(" ")[2]
-        offs = len(mline) - len(text)
-        return [s[offs:] for s in completions if s.startswith(mline)]
-
     def do_set(self, args):
         """
         Set config for cloudmonkey. For example, options can be:
@@ -387,9 +400,12 @@ class CloudMonkeyShell(cmd.Cmd, object):
 
 
 def main():
-    pattern = re.compile("[A-Z]")
-    verbs = list(set([x[:pattern.search(x).start()] for x in completions
-                 if pattern.search(x) is not None]).difference(['cloudstack']))
+    verbs = []
+    if os.path.exists(cache_file):
+        verbs = loadcache(cache_file)['verbs']
+    elif 'verbs' in apicache:
+        verbs = apicache['verbs']
+
     for verb in verbs:
         def add_grammar(verb):
             def grammar_closure(self, args):
@@ -397,9 +413,9 @@ def main():
                     return
                 try:
                     args_partition = args.partition(" ")
-                    res = self.cache_verbs[verb][args_partition[0]]
-                    cmd = res[0]
-                    helpdoc = res[2]
+                    api = self.apicache[verb][args_partition[0]]
+                    cmd = api['name']
+                    helpdoc = api['description']
                     args = args_partition[2]
                 except KeyError, e:
                     self.monkeyprint("Error: invalid %s api arg" % verb, e)
@@ -412,10 +428,10 @@ def main():
 
         grammar_handler = add_grammar(verb)
         grammar_handler.__doc__ = "%ss resources" % verb.capitalize()
-        grammar_handler.__name__ = 'do_' + verb
+        grammar_handler.__name__ = 'do_' + str(verb)
         setattr(CloudMonkeyShell, grammar_handler.__name__, grammar_handler)
 
-    shell = CloudMonkeyShell(sys.argv[0], verbs)
+    shell = CloudMonkeyShell(sys.argv[0])
     if len(sys.argv) > 1:
         shell.onecmd(' '.join(sys.argv[1:]))
     else:

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b5a2e998/tools/cli/cloudmonkey/config.py
----------------------------------------------------------------------
diff --git a/tools/cli/cloudmonkey/config.py b/tools/cli/cloudmonkey/config.py
index b6de641..8b718c2 100644
--- a/tools/cli/cloudmonkey/config.py
+++ b/tools/cli/cloudmonkey/config.py
@@ -26,9 +26,8 @@ try:
 
     from ConfigParser import ConfigParser, SafeConfigParser
     from os.path import expanduser
-    from precache import precached_verbs
 except ImportError, e:
-    precached_verbs = {}
+    print "ImportError", e
 
 param_type = ['boolean', 'date', 'float', 'integer', 'short', 'list',
               'long', 'object', 'map', 'string', 'tzdate', 'uuid']
@@ -37,12 +36,12 @@ iterable_type = ['set', 'list', 'object']
 
 config_dir = expanduser('~/.cloudmonkey')
 config_file = expanduser(config_dir + '/config')
+cache_file = expanduser(config_dir + '/cache')
 
 # cloudmonkey config fields
 config_fields = {'core': {}, 'ui': {}, 'server': {}, 'user': {}}
 
 # core
-config_fields['core']['cache_file'] = expanduser(config_dir + '/cache')
 config_fields['core']['history_file'] = expanduser(config_dir + '/history')
 config_fields['core']['log_file'] = expanduser(config_dir + '/log')
 
@@ -106,10 +105,10 @@ def read_config(get_attr, set_attr):
             try:
                 set_attr(key, config.get(section, key))
             except Exception:
-                missing_keys.appned(key)
+                missing_keys.append(key)
 
     if len(missing_keys) > 0:
-        print "Please fix `%s` in %s" % (key, config_file)
+        print "Please fix `%s` in %s" % (', '.join(missing_keys), config_file)
         sys.exit()
 
     return config_options

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b5a2e998/tools/cli/cloudmonkey/printer.py
----------------------------------------------------------------------
diff --git a/tools/cli/cloudmonkey/printer.py b/tools/cli/cloudmonkey/printer.py
index 40f8473..dd2b8d2 100644
--- a/tools/cli/cloudmonkey/printer.py
+++ b/tools/cli/cloudmonkey/printer.py
@@ -25,7 +25,6 @@ try:
     from pygments.token import *
 
     import sys
-    import types
 except ImportError, e:
     print e
 
@@ -113,21 +112,9 @@ class MonkeyFormatter(Formatter):
                 outfile.write(value)
 
 
-def monkeyprint(color=True, *args):
+def monkeyprint(text):
     fmter = MonkeyFormatter()
     lexer = MonkeyLexer()
     lexer.encoding = 'utf-8'
     fmter.encoding = 'utf-8'
-    output = ""
-    try:
-        for arg in args:
-            if isinstance(type(arg), types.NoneType):
-                continue
-            output += str(arg)
-    except Exception, e:
-        print e
-
-    if color:
-        highlight(output, lexer, fmter, sys.stdout)
-    else:
-        print output
+    highlight(text, lexer, fmter, sys.stdout)

http://git-wip-us.apache.org/repos/asf/incubator-cloudstack/blob/b5a2e998/tools/cli/pom.xml
----------------------------------------------------------------------
diff --git a/tools/cli/pom.xml b/tools/cli/pom.xml
index 582cc57..d99d6fb 100644
--- a/tools/cli/pom.xml
+++ b/tools/cli/pom.xml
@@ -33,45 +33,11 @@
     <defaultGoal>install</defaultGoal>
     <plugins>
       <plugin>
-        <artifactId>maven-antrun-plugin</artifactId>
-        <version>1.7</version>
-        <executions>
-          <execution>
-          <id>generate-resource</id>
-          <phase>generate-resources</phase>
-          <goals>
-            <goal>run</goal>
-          </goals>
-          <configuration>
-            <target>
-              <delete dir="${basedir}/cloudmonkey/marvin"/>
-            </target>
-          </configuration>
-          </execution>
-        </executions>
-      </plugin>
-      <plugin>
         <groupId>org.codehaus.mojo</groupId>
         <artifactId>exec-maven-plugin</artifactId>
         <version>1.2.1</version>
         <executions>
           <execution>
-            <id>compile</id>
-            <phase>compile</phase>
-            <goals>
-              <goal>exec</goal>
-            </goals>
-            <configuration>
-              <workingDirectory>${basedir}</workingDirectory>
-              <executable>cp</executable>
-              <arguments>
-                  <argument>-rv</argument>
-                  <argument>${basedir}/../marvin/marvin</argument>
-                  <argument>${basedir}/cloudmonkey</argument>
-              </arguments>
-            </configuration>
-          </execution>
-          <execution>
             <id>cachemaker</id>
             <phase>compile</phase>
             <goals>


Mime
View raw message