incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tvansteenbu...@apache.org
Subject [14/18] git commit: [#6905] ticket:503 Fix tarball generation
Date Thu, 09 Jan 2014 20:37:01 GMT
[#6905] ticket:503 Fix tarball generation


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/462bcfb5
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/462bcfb5
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/462bcfb5

Branch: refs/heads/tv/6905
Commit: 462bcfb5dbc49cac097f33690446960e1b5c6494
Parents: da8258d
Author: Mykola Kharechko <crchemist@gmail.com>
Authored: Fri Jan 3 14:26:35 2014 +0200
Committer: Tim Van Steenburgh <tvansteenburgh@gmail.com>
Committed: Thu Jan 9 18:41:35 2014 +0000

----------------------------------------------------------------------
 Allura/allura/controllers/repository.py         |  5 +--
 Allura/allura/lib/decorators.py                 |  3 +-
 Allura/allura/model/monq_model.py               |  6 ++++
 Allura/allura/model/repository.py               | 27 +++++++++++---
 Allura/allura/tasks/repo_tasks.py               |  5 +--
 Allura/allura/templates/repo/tarball.html       | 10 +++---
 Allura/development.ini                          |  7 ++++
 .../tests/functional/test_controllers.py        |  4 +--
 .../forgegit/tests/model/test_repository.py     | 37 +++++++++++++++++---
 9 files changed, 83 insertions(+), 21 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/allura/controllers/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/controllers/repository.py b/Allura/allura/controllers/repository.py
index d0e78e2..0dc1274 100644
--- a/Allura/allura/controllers/repository.py
+++ b/Allura/allura/controllers/repository.py
@@ -466,10 +466,11 @@ class CommitBrowser(BaseController):
             raise exc.HTTPNotFound()
         rev = self._commit.url().split('/')[-2]
         status = c.app.repo.get_tarball_status(rev, path)
-        if status is None and request.method == 'POST':
+        if status in (None, 'error') and request.method == 'POST':
             allura.tasks.repo_tasks.tarball.post(revision=rev, path=path)
             redirect('tarball' + '?path={0}'.format(path) if path else '')
-        return dict(commit=self._commit, revision=rev, status=status or 'na')
+        status = 'na' if status in (None, 'error') else status
+        return dict(commit=self._commit, revision=rev, status=status)
 
     @expose('json:')
     def tarball_status(self, path=None, **kw):

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/allura/lib/decorators.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/decorators.py b/Allura/allura/lib/decorators.py
index 8604a92..5810fb0 100644
--- a/Allura/allura/lib/decorators.py
+++ b/Allura/allura/lib/decorators.py
@@ -59,7 +59,8 @@ def task(*args, **kw):
                     kw.get('notifications_disabled') else h.null_contextmanager)
             with cm(project):
                 from allura import model as M
-                return M.MonQTask.post(func, args, kwargs, delay=delay)
+                task_obj = M.MonQTask.post(func, args, kwargs, delay=delay)
+            return task_obj
         # if decorating a class, have to make it a staticmethod
         # or it gets a spurious cls argument
         func.post = staticmethod(post) if inspect.isclass(func) else post

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/allura/model/monq_model.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/monq_model.py b/Allura/allura/model/monq_model.py
index faac14b..f38c073 100644
--- a/Allura/allura/model/monq_model.py
+++ b/Allura/allura/model/monq_model.py
@@ -76,6 +76,11 @@ class MonQTask(MappedClass):
                 # have an index on task_name
                 'state', 'task_name', 'time_queue'
             ],
+            [
+                # used while user quering snapsot
+                'kwargs.revision', 'kwargs.path'
+            ],
+            'args',
         ]
 
     _id = FieldProperty(S.ObjectId)
@@ -246,6 +251,7 @@ class MonQTask(MappedClass):
         old_cuser = getattr(c, 'user', None)
         try:
             func = self.function
+            func.task_id = str(self._id)
             c.project = M.Project.query.get(_id=self.context.project_id)
             c.app = None
             if c.project:

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/allura/model/repository.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/repository.py b/Allura/allura/model/repository.py
index 06ade3f..93e012e 100644
--- a/Allura/allura/model/repository.py
+++ b/Allura/allura/model/repository.py
@@ -39,6 +39,7 @@ import tg
 from paste.deploy.converters import asbool, asint
 from pylons import tmpl_context as c
 from pylons import app_globals as g
+import pymongo
 import pymongo.errors
 
 from ming import schema as S
@@ -56,6 +57,7 @@ from .notification import Notification
 from .repo_refresh import refresh_repo, unknown_commit_ids as unknown_commit_ids_repo
 from .repo import CommitRunDoc, QSIZE
 from .timeline import ActivityObject
+from .monq_model import MonQTask
 
 log = logging.getLogger(__name__)
 config = utils.ConfigProxy(
@@ -355,15 +357,32 @@ class Repository(Artifact, ActivityObject):
                          filename)
         return urljoin(tg.config.get('scm.repos.tarball.url_prefix', '/'), r)
 
-    def get_tarball_status(self, revision, path=None):
+    def get_tarball_status(self, revision, path=None, task_id=None):
         pathname = os.path.join(self.tarball_path, self.tarball_filename(revision, path))
         filename = '%s%s' % (pathname, '.zip')
         tmpfilename = '%s%s' % (pathname, '.tmp')
 
+        # check for state of task in MonQTask
+        path = '' if path is None else path
+        task_query = MonQTask.query.find({
+            'task_name': 'allura.tasks.repo_tasks.tarball',
+            '$or': [
+                {'kwargs.path': path, 'kwargs.revision': revision},
+                {'args':[revision, path]}]})
+        task = task_query.sort(
+            [('time_queue', pymongo.DESCENDING),]).limit(1).first()
+
         if os.path.isfile(filename):
-            return 'ready'
-        elif os.path.isfile(tmpfilename):
-            return 'busy'
+            return 'complete'
+
+        if not task or \
+            (task.state == 'complete' and not os.path.isfile(filename)):
+            return None
+        if task.state == 'busy' and str(task._id) == task_id:
+            return 'self'
+
+        return task.state
+
 
     def __repr__(self): # pragma no cover
         return '<%s %s>' % (

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/allura/tasks/repo_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/repo_tasks.py b/Allura/allura/tasks/repo_tasks.py
index a987228..c4b9296 100644
--- a/Allura/allura/tasks/repo_tasks.py
+++ b/Allura/allura/tasks/repo_tasks.py
@@ -124,8 +124,9 @@ def tarball(revision=None, path=None):
     log = logging.getLogger(__name__)
     if revision:
         repo = c.app.repo
-        status = repo.get_tarball_status(revision, path)
-        if status:
+        status = repo.get_tarball_status(revision, path,
+            task_id=tarball.task_id)
+        if status in ('busy', 'complete'):
             log.info('Skipping snapshot for repository: %s:%s rev %s because it is already
%s' %
                      (c.project.shortname, c.app.config.options.mount_point, revision, status))
         else:

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/allura/templates/repo/tarball.html
----------------------------------------------------------------------
diff --git a/Allura/allura/templates/repo/tarball.html b/Allura/allura/templates/repo/tarball.html
index d68430d..21bc7d7 100644
--- a/Allura/allura/templates/repo/tarball.html
+++ b/Allura/allura/templates/repo/tarball.html
@@ -56,7 +56,7 @@ Commit <a href="{{commit.url()}}">{{commit.shorthand_id()}}</a>
{{commit_labels(
         var delay = 500;
         function check_status() {
             $.get('{{commit.url()}}tarball_status?path={{path}}', function(data) {
-                if (data.status !== 'na') {
+                if (data.status === 'complete') {
                     spinner.stop();
                     $('#snapshot_status h2').hide();
                     $('#snapshot_status h2.' + data.status).show();
@@ -64,13 +64,13 @@ Commit <a href="{{commit.url()}}">{{commit.shorthand_id()}}</a>
{{commit_labels(
                         window.location.href = '{{c.app.repo.tarball_url(revision, path)}}';
                     {% endif %}
                 } else {
+                    if (data.status === 'na' || data.status === 'error') {
+                        // something went wrong
+                        $('#snapshot_status form').show();
+                    }
                     if (delay < 60000){
                         delay = delay * 2;
                     }
-                    if (delay >= 16000) {
-                      // we've been waiting at least 15 seconds
-                      $('#snapshot_status form').show();
-                    }
                     window.setTimeout(check_status, delay);
                 }
             });

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/Allura/development.ini
----------------------------------------------------------------------
diff --git a/Allura/development.ini b/Allura/development.ini
index edfb794..e5ed4f4 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -33,6 +33,13 @@ error_email_from = paste@localhost
 # Used to uniquify references to static resources
 build_key=1276635823
 
+# root directory for tarballs
+scm.repos.tarball.root = /tmp/tarball
+# enable/disable feature
+scm.repos.tarball.enable = false
+# url prefix for tarballs' download urls
+scm.repos.tarball.url_prefix = http://sf-fortytwo-7049.sb.sf.net/p/snapshots/
+
 [server:main]
 use = egg:Paste#http
 host = 0.0.0.0

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/ForgeGit/forgegit/tests/functional/test_controllers.py
----------------------------------------------------------------------
diff --git a/ForgeGit/forgegit/tests/functional/test_controllers.py b/ForgeGit/forgegit/tests/functional/test_controllers.py
index 9a78579..495187a 100644
--- a/ForgeGit/forgegit/tests/functional/test_controllers.py
+++ b/ForgeGit/forgegit/tests/functional/test_controllers.py
@@ -352,9 +352,9 @@ class TestRootController(_TestCase):
         M.MonQTask.run_ready()
         ThreadLocalORMSession.flush_all()
         r = self.app.get(ci + 'tarball_status')
-        assert '{"status": "ready"}' in r
+        assert '{"status": "complete"}' in r
         r = self.app.get('/p/test/src-git/ci/master/tarball_status')
-        assert '{"status": "ready"}' in r
+        assert '{"status": "complete"}' in r
         r = self.app.get('/p/test/src-git/ci/master/tarball')
         assert 'Your download will begin shortly' in r
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/462bcfb5/ForgeGit/forgegit/tests/model/test_repository.py
----------------------------------------------------------------------
diff --git a/ForgeGit/forgegit/tests/model/test_repository.py b/ForgeGit/forgegit/tests/model/test_repository.py
index 4f4f56f..3680846 100644
--- a/ForgeGit/forgegit/tests/model/test_repository.py
+++ b/ForgeGit/forgegit/tests/model/test_repository.py
@@ -24,6 +24,8 @@ import unittest
 import pkg_resources
 import datetime
 
+import pymongo
+
 import mock
 from pylons import tmpl_context as c, app_globals as g
 import tg
@@ -34,6 +36,7 @@ from testfixtures import TempDirectory
 
 from alluratest.controller import setup_basic_test, setup_global_objects
 from allura.lib import helpers as h
+from allura.tasks.repo_tasks import tarball
 from allura.tests import decorators as td
 from allura.tests.model.test_repo import RepoImplTestBase
 from allura import model as M
@@ -394,13 +397,37 @@ class TestGitRepo(unittest.TestCase, RepoImplTestBase):
         if os.path.isdir(os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD/")):
             os.removedirs(os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD/"))
         self.repo.tarball('HEAD')
+        assert_equal(self.repo.get_tarball_status('HEAD'), 'complete')
+
+        os.remove(os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD.zip"))
+        assert_equal(self.repo.get_tarball_status('HEAD'), None)
+
+    def test_tarball_status_task(self):
+        assert_equal(self.repo.get_tarball_status('HEAD'), None)
+
+        # create tarball task in MonQTask and check get_tarball_status
+        tarball.post('HEAD', '')
+
+        # task created
         assert_equal(self.repo.get_tarball_status('HEAD'), 'ready')
-        os.rename(os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD.zip"),
-                  os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD.tmp"))
+
+        task = M.MonQTask.query.find({
+            'task_name': 'allura.tasks.repo_tasks.tarball',
+            '$or': [
+                {'kwargs':
+                    {'path': '',
+                     'revision': 'HEAD'}},
+                    {'args':['HEAD', '']}]
+            }).sort([('time_queue', pymongo.DESCENDING),]).limit(1).first()
+
+        # task is running
+        task.state = 'busy'
+        task.query.session.flush_all()
         assert_equal(self.repo.get_tarball_status('HEAD'), 'busy')
-        os.remove(os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD.tmp"))
-        assert_equal(self.repo.get_tarball_status('HEAD'), None)
-        os.makedirs(os.path.join(tmpdir, "git/t/te/test/testgit.git/test-src-git-HEAD"))
+
+        # when state is complete, but file don't exists, then status is None
+        task.state = 'complete'
+        task.query.session.flush_all()
         assert_equal(self.repo.get_tarball_status('HEAD'), None)
 
     def test_is_empty(self):


Mime
View raw message