incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tvansteenbu...@apache.org
Subject git commit: [#5703] Refactor taskable script pattern
Date Tue, 12 Feb 2013 21:59:32 GMT
Updated Branches:
  refs/heads/tv/5703 683419d58 -> 5c82af948


[#5703] Refactor taskable script pattern

- Apply @task decorator to class itself
- Generate class doc string from arg parser
- Remove --clean-all option from repo refresh script
- Remove session.close_all() from repo refresh script (breaks task)
- Remove old copy of repo refresh script


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/5c82af94
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/5c82af94
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/5c82af94

Branch: refs/heads/tv/5703
Commit: 5c82af94874b23102df92ce3622573adcc746652
Parents: 683419d
Author: Tim Van Steenburgh <tvansteenburgh@gmail.com>
Authored: Tue Feb 12 21:55:01 2013 +0000
Committer: Tim Van Steenburgh <tvansteenburgh@gmail.com>
Committed: Tue Feb 12 21:55:01 2013 +0000

----------------------------------------------------------------------
 Allura/allura/lib/decorators.py      |   12 ++-
 Allura/allura/scripts/refreshrepo.py |   12 --
 Allura/allura/scripts/scripttask.py  |   31 ++----
 scripts/refresh-all-repos.py         |  169 -----------------------------
 4 files changed, 22 insertions(+), 202 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5c82af94/Allura/allura/lib/decorators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/decorators.py b/Allura/allura/lib/decorators.py
index bd26b1e..c97230f 100644
--- a/Allura/allura/lib/decorators.py
+++ b/Allura/allura/lib/decorators.py
@@ -1,3 +1,4 @@
+import inspect
 import sys
 import json
 import logging
@@ -12,12 +13,21 @@ from webob import exc
 
 def task(func):
     '''Decorator to add some methods to task functions'''
+    if inspect.isclass(func):
+        return taskclass(func)
     def post(*args, **kwargs):
         from allura import model as M
         return M.MonQTask.post(func, args, kwargs)
     func.post = post
     return func
 
+def taskclass(cls):
+    def post(*args, **kwargs):
+        from allura import model as M
+        return M.MonQTask.post(cls, args[1:], kwargs)
+    cls.post = classmethod(post)
+    return cls
+
 class event_handler(object):
     '''Decorator to register event handlers'''
     listeners = defaultdict(set)
@@ -195,4 +205,4 @@ def memoize(func, *args):
     else:
         result = func(*args)
         dic[args] = result
-        return result
\ No newline at end of file
+        return result

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5c82af94/Allura/allura/scripts/refreshrepo.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/refreshrepo.py b/Allura/allura/scripts/refreshrepo.py
index 4124742..09b6242 100644
--- a/Allura/allura/scripts/refreshrepo.py
+++ b/Allura/allura/scripts/refreshrepo.py
@@ -28,14 +28,6 @@ class RefreshRepo(ScriptTask):
             q_project['shortname'] = {'$regex': options.project_regex}
 
         log.info('Refreshing repositories')
-        if options.clean_all:
-            log.info('Removing all repository objects')
-            M.repo.CommitDoc.m.remove({})
-            M.repo.TreeDoc.m.remove({})
-            M.repo.TreesDoc.m.remove({})
-            M.repo.DiffInfoDoc.m.remove({})
-            M.repo.CommitRunDoc.m.remove({})
-
         for chunk in chunked_find(M.Project, q_project):
             for p in chunk:
                 log.info("Refreshing repos for project '%s'." % p.shortname)
@@ -115,7 +107,6 @@ class RefreshRepo(ScriptTask):
                     except:
                         log.exception('Error refreshing %r', c.app.repo)
             ThreadLocalORMSession.flush_all()
-            ThreadLocalORMSession.close_all()
 
     @classmethod
     def parser(cls):
@@ -150,9 +141,6 @@ class RefreshRepo(ScriptTask):
         parser.add_argument('--clean', action='store_true', dest='clean',
                 default=False, help='Remove repo-related mongo docs (for '
                 'project(s) being refreshed only) before doing the refresh.')
-        parser.add_argument('--clean-all', action='store_true', dest='clean_all',
-                default=False, help='Remove ALL repo-related mongo docs before '
-                'refresh.')
         parser.add_argument('--all', action='store_true', dest='all', default=False,
                 help='Refresh all commits (not just the ones that are new).')
         parser.add_argument('--notify', action='store_true', dest='notify',

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5c82af94/Allura/allura/scripts/scripttask.py
----------------------------------------------------------------------
diff --git a/Allura/allura/scripts/scripttask.py b/Allura/allura/scripts/scripttask.py
index 4ce2dc2..9dbe857 100644
--- a/Allura/allura/scripts/scripttask.py
+++ b/Allura/allura/scripts/scripttask.py
@@ -28,11 +28,8 @@ To call as a task::
 """
 
 import argparse
-import copy_reg
 import logging
-import pickle
 import sys
-import types
 
 from allura.lib.decorators import task
 
@@ -40,18 +37,6 @@ from allura.lib.decorators import task
 log = logging.getLogger(__name__)
 
 
-# make methods picklable
-def reduce_method(m):
-    return (getattr, (m.__self__, m.__func__.__name__))
-copy_reg.pickle(types.MethodType, reduce_method)
-
-
-@task
-def dispatcher(pickled_method, *args, **kw):
-    method = pickle.loads(pickled_method)
-    return method(*args, **kw)
-
-
 class Writer(object):
     def __init__(self, func):
         self.func = func
@@ -62,6 +47,17 @@ class Writer(object):
 
 class ScriptTask(object):
     """Base class for a command-line script that is also executable as a task."""
+
+    class __metaclass__(type):
+        @property
+        def __doc__(cls):
+            return cls.parser().format_help()
+        def __new__(meta, classname, bases, classDict):
+            return task(type.__new__(meta, classname, bases, classDict))
+
+    def __new__(cls, arg_string):
+        cls._execute_task(arg_string)
+
     @classmethod
     def _execute_task(cls, arg_string):
         try:
@@ -89,11 +85,6 @@ class ScriptTask(object):
         return argparse.ArgumentParser(description="Default no-op parser")
 
     @classmethod
-    def post(cls, arg_string=''):
-        pickled_method = pickle.dumps(cls._execute_task)
-        return dispatcher.post(pickled_method, arg_string)
-
-    @classmethod
     def main(cls):
         options = cls.parser().parse_args()
         cls.execute(options)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/5c82af94/scripts/refresh-all-repos.py
----------------------------------------------------------------------
diff --git a/scripts/refresh-all-repos.py b/scripts/refresh-all-repos.py
deleted file mode 100644
index 822148f..0000000
--- a/scripts/refresh-all-repos.py
+++ /dev/null
@@ -1,169 +0,0 @@
-import argparse
-import logging
-import re
-
-import faulthandler
-from pylons import c
-from ming.orm import ThreadLocalORMSession
-
-from allura import model as M
-from allura.lib.utils import chunked_find, chunked_list
-
-log = logging.getLogger(__name__)
-
-
-def main(options):
-    q_project = {}
-    if options.nbhd:
-        nbhd = M.Neighborhood.query.get(url_prefix=options.nbhd)
-        if not nbhd:
-            return "Invalid neighborhood url prefix."
-        q_project['neighborhood_id'] = nbhd._id
-    if options.project:
-        q_project['shortname'] = options.project
-    elif options.project_regex:
-        q_project['shortname'] = {'$regex': options.project_regex}
-
-    log.info('Refreshing repositories')
-    if options.clean_all:
-        log.info('Removing all repository objects')
-        M.repo.CommitDoc.m.remove({})
-        M.repo.TreeDoc.m.remove({})
-        M.repo.TreesDoc.m.remove({})
-        M.repo.DiffInfoDoc.m.remove({})
-        M.repo.CommitRunDoc.m.remove({})
-
-    for chunk in chunked_find(M.Project, q_project):
-        for p in chunk:
-            log.info("Refreshing repos for project '%s'." % p.shortname)
-            if options.dry_run:
-                continue
-            c.project = p
-            if options.mount_point:
-                mount_points = [options.mount_point]
-            else:
-                mount_points = [ac.options.mount_point for ac in
-                                M.AppConfig.query.find(dict(project_id=p._id))]
-            for app in (p.app_instance(mp) for mp in mount_points):
-                c.app = app
-                if not hasattr(app, 'repo'):
-                    continue
-                if c.app.repo.tool.lower() not in options.repo_types:
-                    log.info("Skipping %r: wrong type (%s)", c.app.repo,
-                            c.app.repo.tool.lower())
-                    continue
-
-                if options.clean:
-                    ci_ids = list(c.app.repo.all_commit_ids())
-                    log.info("Deleting mongo data for %i commits...", len(ci_ids))
-                    tree_ids = [
-                            tree_id for doc in
-                            M.repo.TreesDoc.m.find({"_id": {"$in": ci_ids}},
-                                                   {"tree_ids": 1})
-                            for tree_id in doc.get("tree_ids", [])]
-
-                    i = M.repo.CommitDoc.m.find({"_id": {"$in": ci_ids}}).count()
-                    log.info("Deleting %i CommitDoc docs...", i)
-                    M.repo.CommitDoc.m.remove({"_id": {"$in": ci_ids}})
-
-                    # delete these in chunks, otherwise the query doc can
-                    # exceed the max BSON size limit (16MB at the moment)
-                    for tree_ids_chunk in chunked_list(tree_ids, 300000):
-                        i = M.repo.TreeDoc.m.find({"_id": {"$in": tree_ids_chunk}}).count()
-                        log.info("Deleting %i TreeDoc docs...", i)
-                        M.repo.TreeDoc.m.remove({"_id": {"$in": tree_ids_chunk}})
-                        i = M.repo.LastCommitDoc.m.find({"object_id": {"$in": tree_ids_chunk}}).count()
-                        log.info("Deleting %i LastCommitDoc docs...", i)
-                        M.repo.LastCommitDoc.m.remove({"object_id": {"$in": tree_ids_chunk}})
-                    del tree_ids
-
-                    # delete these after TreeDoc and LastCommitDoc so that if
-                    # we crash, we don't lose the ability to delete those
-                    i = M.repo.TreesDoc.m.find({"_id": {"$in": ci_ids}}).count()
-                    log.info("Deleting %i TreesDoc docs...", i)
-                    M.repo.TreesDoc.m.remove({"_id": {"$in": ci_ids}})
-
-                    # delete LastCommitDocs for non-trees
-                    repo_lastcommit_re = re.compile("^{}:".format(c.app.repo._id))
-                    i = M.repo.LastCommitDoc.m.find(dict(_id=repo_lastcommit_re)).count()
-                    log.info("Deleting %i remaining LastCommitDoc docs, by repo id...", i)
-                    M.repo.LastCommitDoc.m.remove(dict(_id=repo_lastcommit_re))
-
-                    i = M.repo.DiffInfoDoc.m.find({"_id": {"$in": ci_ids}}).count()
-                    log.info("Deleting %i DiffInfoDoc docs...", i)
-                    M.repo.DiffInfoDoc.m.remove({"_id": {"$in": ci_ids}})
-
-                    i = M.repo.CommitRunDoc.m.find({"commit_ids": {"$in": ci_ids}}).count()
-                    log.info("Deleting %i CommitRunDoc docs...", i)
-                    M.repo.CommitRunDoc.m.remove({"commit_ids": {"$in": ci_ids}})
-                    del ci_ids
-
-                try:
-                    if options.all:
-                        log.info('Refreshing ALL commits in %r', c.app.repo)
-                    else:
-                        log.info('Refreshing NEW commits in %r', c.app.repo)
-                    if options.profile:
-                        import cProfile
-                        cProfile.runctx('c.app.repo.refresh(options.all, notify=options.notify)',
-                                globals(), locals(), 'refresh.profile')
-                    else:
-                        c.app.repo.refresh(options.all, notify=options.notify)
-                except:
-                    log.exception('Error refreshing %r', c.app.repo)
-        ThreadLocalORMSession.flush_all()
-        ThreadLocalORMSession.close_all()
-
-
-def repo_type_list(s):
-    repo_types = []
-    for repo_type in s.split(','):
-        repo_type = repo_type.strip()
-        if repo_type not in ['svn', 'git', 'hg']:
-            raise argparse.ArgumentTypeError(
-                    '{} is not a valid repo type.'.format(repo_type))
-        repo_types.append(repo_type)
-    return repo_types
-
-
-def parse_options():
-    parser = argparse.ArgumentParser(description='Scan repos on filesytem and '
-            'update repo metadata in MongoDB. Run for all repos (no args), '
-            'or restrict by neighborhood, project, or code tool mount point.')
-    parser.add_argument('--nbhd', action='store', default='', dest='nbhd',
-            help='Restrict update to a particular neighborhood, e.g. /p/.')
-    parser.add_argument('--project', action='store', default='', dest='project',
-            help='Restrict update to a particular project. To specify a '
-            'subproject, use a slash: project/subproject.')
-    parser.add_argument('--project-regex', action='store', default='',
-            dest='project_regex',
-            help='Restrict update to projects for which the shortname matches '
-            'the provided regex.')
-    parser.add_argument('--repo-types', action='store', type=repo_type_list,
-            default=['svn', 'git', 'hg'], dest='repo_types',
-            help='Only refresh repos of the given type(s). Defaults to: '
-            'svn,git,hg. Example: --repo-types=git,hg')
-    parser.add_argument('--mount_point', default='', dest='mount_point',
-            help='Restrict update to repos at the given tool mount point. ')
-    parser.add_argument('--clean', action='store_true', dest='clean',
-            default=False, help='Remove repo-related mongo docs (for '
-            'project(s) being refreshed only) before doing the refresh.')
-    parser.add_argument('--clean-all', action='store_true', dest='clean_all',
-            default=False, help='Remove ALL repo-related mongo docs before '
-            'refresh.')
-    parser.add_argument('--all', action='store_true', dest='all', default=False,
-            help='Refresh all commits (not just the ones that are new).')
-    parser.add_argument('--notify', action='store_true', dest='notify',
-            default=False, help='Send email notifications of new commits.')
-    parser.add_argument('--dry-run', action='store_true', dest='dry_run',
-            default=False, help='Log names of projects that would have their '
-            'repos refreshed, but do not perform the actual refresh.')
-    parser.add_argument('--profile', action='store_true', dest='profile',
-            default=False, help='Enable the profiler (slow). Will log '
-            'profiling output to ./refresh.profile')
-    return parser.parse_args()
-
-if __name__ == '__main__':
-    import sys
-    faulthandler.enable()
-    sys.exit(main(parse_options()))


Mime
View raw message