incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject [20/20] git commit: [#6540] Added rate limiting on tool imports
Date Tue, 10 Sep 2013 15:11:21 GMT
[#6540] Added rate limiting on tool imports

Signed-off-by: Cory Johns <cjohns@slashdotmedia.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/51ae3873
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/51ae3873
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/51ae3873

Branch: refs/heads/cj/6540
Commit: 51ae3873467dadcb912595d13ee5131597668346
Parents: 5445e84
Author: Cory Johns <cjohns@slashdotmedia.com>
Authored: Mon Sep 9 22:35:20 2013 +0000
Committer: Cory Johns <cjohns@slashdotmedia.com>
Committed: Tue Sep 10 15:09:22 2013 +0000

----------------------------------------------------------------------
 ForgeImporters/forgeimporters/base.py           | 44 +++++++++++++++++++-
 ForgeImporters/forgeimporters/github/code.py    | 19 +++++----
 .../forgeimporters/github/tests/test_code.py    | 20 +++++++++
 ForgeImporters/forgeimporters/google/code.py    | 17 ++++----
 .../forgeimporters/google/tests/test_code.py    | 18 ++++++++
 ForgeImporters/forgeimporters/google/tracker.py | 19 +++++----
 .../forgeimporters/tests/google/test_tracker.py | 20 ++++++++-
 .../forgeimporters/trac/tests/test_tickets.py   | 18 ++++++++
 ForgeImporters/forgeimporters/trac/tickets.py   | 19 +++++----
 9 files changed, 160 insertions(+), 34 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/51ae3873/ForgeImporters/forgeimporters/base.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/base.py b/ForgeImporters/forgeimporters/base.py
index ffb1650..d7479b3 100644
--- a/ForgeImporters/forgeimporters/base.py
+++ b/ForgeImporters/forgeimporters/base.py
@@ -68,14 +68,16 @@ class ToolImportForm(schema.Schema):
 
 
 class ImportErrorHandler(object):
-    def __init__(self, importer, project_name):
+    def __init__(self, importer, project_name, project):
         self.importer = importer
         self.project_name = project_name
+        self.project = project
 
     def __enter__(self):
         pass
 
     def __exit__(self, exc_type, exc_val, exc_tb):
+        self.importer.clear_pending(self.project)
         if exc_type:
             g.post_event('import_tool_task_failed',
                 error=str(exc_val),
@@ -89,7 +91,7 @@ class ImportErrorHandler(object):
 @task(notifications_disabled=True)
 def import_tool(importer_name, project_name=None, mount_point=None, mount_label=None, **kw):
     importer = ToolImporter.by_name(importer_name)
-    with ImportErrorHandler(importer, project_name):
+    with ImportErrorHandler(importer, project_name, c.project):
         importer.import_tool(c.project, c.user, project_name=project_name,
                 mount_point=mount_point, mount_label=mount_label, **kw)
 
@@ -326,6 +328,44 @@ class ToolImporter(object):
                 importers[ep.name] = importer()
         return importers
 
+    @property
+    def classname(self):
+        return self.__class__.__name__
+
+    def enforce_limit(self, project):
+        """
+        Enforce rate limiting of tool imports on a given project.
+
+        Returns False if limit is met / exceeded.  Otherwise, increments the
+        count of pending / in-progress imports and returns True.
+        """
+        limit = config.get('tool_import.rate_limit', 1)
+        pending_key = 'tool_data.%s.pending' % self.classname
+        modified_project = M.Project.query.find_and_modify(
+                query={
+                        '_id': project._id,
+                        '$or': [
+                                {pending_key: None},
+                                {pending_key: {'$lt': limit}},
+                            ],
+                    },
+                update={'$inc': {pending_key: 1}},
+                new=True,
+            )
+        return modified_project is not None
+
+    def clear_pending(self, project):
+        """
+        Decrement the pending counter for this importer on the given project,
+        to indicate that an import is complete.
+        """
+        pending_key = 'tool_data.%s.pending' % self.classname
+        M.Project.query.find_and_modify(
+                query={'_id': project._id},
+                update={'$inc': {pending_key: -1}},
+                new=True,
+            )
+
     def import_tool(self, project, user, project_name=None,
             mount_point=None, mount_label=None, **kw):
         """

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/51ae3873/ForgeImporters/forgeimporters/github/code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/code.py b/ForgeImporters/forgeimporters/github/code.py
index 92e5626..aa5fe7d 100644
--- a/ForgeImporters/forgeimporters/github/code.py
+++ b/ForgeImporters/forgeimporters/github/code.py
@@ -45,7 +45,7 @@ from forgeimporters.github import GitHubProjectExtractor
 @task(notifications_disabled=True)
 def import_tool(**kw):
     importer = GitHubRepoImporter()
-    with ImportErrorHandler(importer, kw.get('project_name')):
+    with ImportErrorHandler(importer, kw.get('project_name'), c.project):
         importer.import_tool(c.project, c.user, **kw)
 
 
@@ -73,13 +73,16 @@ class GitHubRepoImportController(BaseController):
     @require_post()
     @validate(GitHubRepoImportForm(ForgeGitApp), error_handler=index)
     def create(self, gh_project_name, gh_user_name, mount_point, mount_label, **kw):
-        import_tool.post(
-                project_name=gh_project_name,
-                user_name=gh_user_name,
-                mount_point=mount_point,
-                mount_label=mount_label)
-        flash('Repo import has begun. Your new repo will be available '
-                'when the import is complete.')
+        if GitHubRepoImporter().enforce_limit(c.project):
+            import_tool.post(
+                    project_name=gh_project_name,
+                    user_name=gh_user_name,
+                    mount_point=mount_point,
+                    mount_label=mount_label)
+            flash('Repo import has begun. Your new repo will be available '
+                    'when the import is complete.')
+        else:
+            flash('There are too many imports pending at this time.  Please wait and try
again.', 'error')
         redirect(c.project.url() + 'admin/')
 
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/51ae3873/ForgeImporters/forgeimporters/github/tests/test_code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/github/tests/test_code.py b/ForgeImporters/forgeimporters/github/tests/test_code.py
index 09afa87..729d8c5 100644
--- a/ForgeImporters/forgeimporters/github/tests/test_code.py
+++ b/ForgeImporters/forgeimporters/github/tests/test_code.py
@@ -17,9 +17,11 @@
 
 from unittest import TestCase
 from mock import Mock, patch
+from ming.odm import ThreadLocalORMSession
 
 from allura.tests import TestController
 from allura.tests.decorators import with_tool
+from allura import model as M
 from forgeimporters.github.code import GitHubRepoImporter
 
 
@@ -76,3 +78,21 @@ class TestGitHubImportController(TestController, TestCase):
         self.assertEqual(u'mylabel', import_tool.post.call_args[1]['mount_label'])
         self.assertEqual(u'poop', import_tool.post.call_args[1]['project_name'])
         self.assertEqual(u'spooky', import_tool.post.call_args[1]['user_name'])
+
+    @with_git
+    @patch('forgeimporters.github.code.import_tool')
+    def test_create_limit(self, import_tool):
+        project = M.Project.query.get(shortname=test_project_with_repo)
+        project.set_tool_data('GitHubRepoImporter', pending=1)
+        ThreadLocalORMSession.flush_all()
+        params = dict(
+                gh_user_name='spooky',
+                gh_project_name='poop',
+                mount_label='mylabel',
+                mount_point='mymount',
+                )
+        r = self.app.post('/p/{}/admin/ext/import/github-repo/create'.format(test_project_with_repo),
+                params,
+                status=302).follow()
+        self.assertIn('Please wait and try again', r)
+        self.assertEqual(import_tool.post.call_count, 0)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/51ae3873/ForgeImporters/forgeimporters/google/code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/code.py b/ForgeImporters/forgeimporters/google/code.py
index 134ecb6..f8e42b3 100644
--- a/ForgeImporters/forgeimporters/google/code.py
+++ b/ForgeImporters/forgeimporters/google/code.py
@@ -88,7 +88,7 @@ def get_repo_class(type_):
 @task(notifications_disabled=True)
 def import_tool(**kw):
     importer = GoogleRepoImporter()
-    with ImportErrorHandler(importer, kw.get('project_name')):
+    with ImportErrorHandler(importer, kw.get('project_name'), c.project):
         importer.import_tool(c.project, c.user, **kw)
 
 
@@ -140,12 +140,15 @@ class GoogleRepoImportController(BaseController):
     @require_post()
     @validate(GoogleRepoImportForm(), error_handler=index)
     def create(self, gc_project_name, mount_point, mount_label, **kw):
-        import_tool.post(
-                project_name=gc_project_name,
-                mount_point=mount_point,
-                mount_label=mount_label)
-        flash('Repo import has begun. Your new repo will be available '
-                'when the import is complete.')
+        if GoogleRepoImporter().enforce_limit(c.project):
+            import_tool.post(
+                    project_name=gc_project_name,
+                    mount_point=mount_point,
+                    mount_label=mount_label)
+            flash('Repo import has begun. Your new repo will be available '
+                    'when the import is complete.')
+        else:
+            flash('There are too many imports pending at this time.  Please wait and try
again.', 'error')
         redirect(c.project.url() + 'admin/')
 
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/51ae3873/ForgeImporters/forgeimporters/google/tests/test_code.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/tests/test_code.py b/ForgeImporters/forgeimporters/google/tests/test_code.py
index 806a004..96dd687 100644
--- a/ForgeImporters/forgeimporters/google/tests/test_code.py
+++ b/ForgeImporters/forgeimporters/google/tests/test_code.py
@@ -17,9 +17,11 @@
 
 from unittest import TestCase
 from mock import Mock, patch
+from ming.odm import ThreadLocalORMSession
 
 from allura.tests import TestController
 from allura.tests.decorators import with_tool
+from allura import model as M
 
 
 # important to be distinct from 'test' which ForgeSVN uses, so that the tests can run in
parallel and not clobber each other
@@ -113,3 +115,19 @@ class TestGoogleRepoImportController(TestController, TestCase):
         self.assertEqual(u'mymount', import_tool.post.call_args[1]['mount_point'])
         self.assertEqual(u'mylabel', import_tool.post.call_args[1]['mount_label'])
         self.assertEqual(u'poop', import_tool.post.call_args[1]['project_name'])
+
+    @with_svn
+    @patch('forgeimporters.google.code.import_tool')
+    def test_create_limit(self, import_tool):
+        project = M.Project.query.get(shortname=test_project_with_repo)
+        project.set_tool_data('GoogleRepoImporter', pending=1)
+        ThreadLocalORMSession.flush_all()
+        params = dict(gc_project_name='poop',
+                mount_label='mylabel',
+                mount_point='mymount',
+                )
+        r = self.app.post('/p/{}/admin/src/_importer/create'.format(test_project_with_repo),
+                params,
+                status=302).follow()
+        self.assertIn('Please wait and try again', r)
+        self.assertEqual(import_tool.post.call_count, 0)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/51ae3873/ForgeImporters/forgeimporters/google/tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/google/tracker.py b/ForgeImporters/forgeimporters/google/tracker.py
index 9c259dd..d52945e 100644
--- a/ForgeImporters/forgeimporters/google/tracker.py
+++ b/ForgeImporters/forgeimporters/google/tracker.py
@@ -54,7 +54,7 @@ from forgeimporters.base import (
 @task(notifications_disabled=True)
 def import_tool(**kw):
     importer = GoogleCodeTrackerImporter()
-    with ImportErrorHandler(importer, kw.get('project_name')):
+    with ImportErrorHandler(importer, kw.get('project_name'), c.project):
         importer.import_tool(c.project, c.user, **kw)
 
 
@@ -81,13 +81,16 @@ class GoogleCodeTrackerImportController(BaseController):
     @require_post()
     @validate(GoogleCodeTrackerImportForm(ForgeTrackerApp), error_handler=index)
     def create(self, gc_project_name, mount_point, mount_label, **kw):
-        import_tool.post(
-                project_name=gc_project_name,
-                mount_point=mount_point,
-                mount_label=mount_label,
-                )
-        flash('Ticket import has begun. Your new tracker will be available '
-                'when the import is complete.')
+        if GoogleCodeTrackerImporter().enforce_limit(c.project):
+            import_tool.post(
+                    project_name=gc_project_name,
+                    mount_point=mount_point,
+                    mount_label=mount_label,
+                    )
+            flash('Ticket import has begun. Your new tracker will be available '
+                    'when the import is complete.')
+        else:
+            flash('There are too many imports pending at this time.  Please wait and try
again.', 'error')
         redirect(c.project.url() + 'admin/')
 
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/51ae3873/ForgeImporters/forgeimporters/tests/google/test_tracker.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/tests/google/test_tracker.py b/ForgeImporters/forgeimporters/tests/google/test_tracker.py
index d795bab..98216f9 100644
--- a/ForgeImporters/forgeimporters/tests/google/test_tracker.py
+++ b/ForgeImporters/forgeimporters/tests/google/test_tracker.py
@@ -17,13 +17,16 @@
 
 from datetime import datetime
 from unittest import TestCase
+
 import mock
 from mock import patch
+from ming.odm import ThreadLocalORMSession
 
 from allura.tests import TestController
 from allura.tests.decorators import with_tracker
 
-from ...google import tracker
+from allura import model as M
+from forgeimporters.google import tracker
 
 
 class TestTrackerImporter(TestCase):
@@ -299,3 +302,18 @@ class TestGoogleCodeTrackerImportController(TestController, TestCase):
         self.assertEqual(u'mymount', import_tool.post.call_args[1]['mount_point'])
         self.assertEqual(u'mylabel', import_tool.post.call_args[1]['mount_label'])
         self.assertEqual(u'test', import_tool.post.call_args[1]['project_name'])
+
+    @with_tracker
+    @patch('forgeimporters.google.tracker.import_tool')
+    def test_create_limit(self, import_tool):
+        project = M.Project.query.get(shortname='test')
+        project.set_tool_data('GoogleCodeTrackerImporter', pending=1)
+        ThreadLocalORMSession.flush_all()
+        params = dict(gc_project_name='test',
+                mount_label='mylabel',
+                mount_point='mymount',
+                )
+        r = self.app.post('/p/test/admin/bugs/_importer/create', params,
+                status=302).follow()
+        self.assertIn('Please wait and try again', r)
+        self.assertEqual(import_tool.post.call_count, 0)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/51ae3873/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
index 92e7854..b3d2fcb 100644
--- a/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tests/test_tickets.py
@@ -18,12 +18,14 @@
 import json
 from unittest import TestCase
 from mock import Mock, patch
+from ming.orm import ThreadLocalORMSession
 
 from tg import config
 
 from allura.tests import TestController
 from allura.tests.decorators import with_tracker
 
+from allura import model as M
 from forgeimporters.trac.tickets import (
     TracTicketImporter,
     TracTicketImportController,
@@ -132,3 +134,19 @@ class TestTracTicketImportController(TestController, TestCase):
         self.assertEqual(u'mylabel', import_tool.post.call_args[1]['mount_label'])
         self.assertEqual('{"orig_user": "new_user"}', import_tool.post.call_args[1]['user_map'])
         self.assertEqual(u'http://example.com/trac/url', import_tool.post.call_args[1]['trac_url'])
+
+    @with_tracker
+    @patch('forgeimporters.trac.tickets.import_tool')
+    def test_create_limit(self, import_tool):
+        project = M.Project.query.get(shortname='test')
+        project.set_tool_data('TracTicketImporter', pending=1)
+        ThreadLocalORMSession.flush_all()
+        params = dict(trac_url='http://example.com/trac/url',
+                mount_label='mylabel',
+                mount_point='mymount',
+                )
+        r = self.app.post('/p/test/admin/bugs/_importer/create', params,
+                upload_files=[('user_map', 'myfile', '{"orig_user": "new_user"}')],
+                status=302).follow()
+        self.assertIn('Please wait and try again', r)
+        self.assertEqual(import_tool.post.call_count, 0)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/51ae3873/ForgeImporters/forgeimporters/trac/tickets.py
----------------------------------------------------------------------
diff --git a/ForgeImporters/forgeimporters/trac/tickets.py b/ForgeImporters/forgeimporters/trac/tickets.py
index 9e4f493..cafd403 100644
--- a/ForgeImporters/forgeimporters/trac/tickets.py
+++ b/ForgeImporters/forgeimporters/trac/tickets.py
@@ -61,7 +61,7 @@ from forgetracker.scripts.import_tracker import import_tracker
 @task(notifications_disabled=True)
 def import_tool(**kw):
     importer = TracTicketImporter()
-    with ImportErrorHandler(importer, kw.get('trac_url')):
+    with ImportErrorHandler(importer, kw.get('trac_url'), c.project):
         importer.import_tool(c.project, c.user, **kw)
 
 
@@ -89,13 +89,16 @@ class TracTicketImportController(BaseController):
     @require_post()
     @validate(TracTicketImportForm(ForgeTrackerApp), error_handler=index)
     def create(self, trac_url, mount_point, mount_label, user_map=None, **kw):
-        import_tool.post(
-                mount_point=mount_point,
-                mount_label=mount_label,
-                trac_url=trac_url,
-                user_map=user_map)
-        flash('Ticket import has begun. Your new tracker will be available '
-                'when the import is complete.')
+        if TracTicketImporter().enforce_limit(c.project):
+            import_tool.post(
+                    mount_point=mount_point,
+                    mount_label=mount_label,
+                    trac_url=trac_url,
+                    user_map=user_map)
+            flash('Ticket import has begun. Your new tracker will be available '
+                    'when the import is complete.')
+        else:
+            flash('There are too many imports pending at this time.  Please wait and try
again.', 'error')
         redirect(c.project.url() + 'admin/')
 
 


Mime
View raw message