allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From brond...@apache.org
Subject [2/2] git commit: [#4397] Background timeline aggregation
Date Wed, 08 Jan 2014 18:35:39 GMT
[#4397] Background timeline aggregation

Fires off background tasks to aggregate affected timelines after a new
activity is created. Multiple aggregations for the same node are
prevented from occuring concurrently. On-demand aggregation will still
happen if necessary (if a timeline is requested and it's aggregation
is stale) but should be rare.

Signed-off-by: Tim Van Steenburgh <tvansteenburgh@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/dca06665
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/dca06665
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/dca06665

Branch: refs/heads/master
Commit: dca066651ec43fb6541075bacda767151fb7562c
Parents: 131f06f
Author: Tim Van Steenburgh <tvansteenburgh@gmail.com>
Authored: Thu Dec 19 02:45:59 2013 +0000
Committer: Dave Brondsema <dbrondsema@slashdotmedia.com>
Committed: Wed Jan 8 18:35:26 2014 +0000

----------------------------------------------------------------------
 Allura/allura/lib/app_globals.py                |  4 +++
 Allura/allura/lib/custom_middleware.py          |  6 ++++
 Allura/allura/model/timeline.py                 | 36 ++++++++++++++++++--
 Allura/allura/tasks/activity_tasks.py           | 25 ++++++++++++++
 Allura/setup.py                                 |  6 ++--
 .../forgeactivity/tests/functional/test_root.py | 22 ++++++++++++
 6 files changed, 94 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dca06665/Allura/allura/lib/app_globals.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 0643460..6c94870 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -280,6 +280,10 @@ class Globals(object):
                     return False
                 def create_activity(self, *a, **kw):
                     pass
+                def create_timeline(self, *a, **kw):
+                    pass
+                def create_timelines(self, *a, **kw):
+                    pass
                 def get_timeline(self, *a, **kw):
                     return []
             return NullActivityStreamDirector()

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dca06665/Allura/allura/lib/custom_middleware.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/custom_middleware.py b/Allura/allura/lib/custom_middleware.py
index d7cbdc9..940c534 100644
--- a/Allura/allura/lib/custom_middleware.py
+++ b/Allura/allura/lib/custom_middleware.py
@@ -188,8 +188,14 @@ class AlluraTimerMiddleware(TimerMiddleware):
         import pymongo
         import socket
         import urllib2
+        import activitystream
 
         return self.entry_point_timers() + [
+            Timer('activitystream.director.{method_name}', allura.model.timeline.Director,
+                'create_activity', 'create_timeline', 'get_timeline'),
+            Timer('activitystream.aggregator.{method_name}', allura.model.timeline.Aggregator,
'*'),
+            Timer('activitystream.node_manager.{method_name}', activitystream.managers.NodeManager,
'*'),
+            Timer('activitystream.activity_manager.{method_name}', activitystream.managers.ActivityManager,
'*'),
             Timer('jinja', jinja2.Template, 'render', 'stream', 'generate'),
             Timer('markdown', markdown.Markdown, 'convert'),
             Timer('ming', ming.odm.odmsession.ODMCursor, 'next',  # FIXME: this may captures
timings ok, but is misleading for counts

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dca06665/Allura/allura/model/timeline.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/timeline.py b/Allura/allura/model/timeline.py
index b45118e..8f8ac41 100644
--- a/Allura/allura/model/timeline.py
+++ b/Allura/allura/model/timeline.py
@@ -16,18 +16,48 @@
 #       under the License.
 
 import bson
+import logging
 from ming.odm import Mapper
-from activitystream import base
+
+from activitystream import ActivityDirector
+from activitystream.base import NodeBase, ActivityObjectBase
+from activitystream.managers import Aggregator as BaseAggregator
+
 from allura.lib import security
+from allura.tasks.activity_tasks import create_timelines
+
+log = logging.getLogger(__name__)
+
+
+class Director(ActivityDirector):
+    """Overrides the default ActivityDirector to kick off background
+    timeline aggregations after an activity is created.
+
+    """
+    def create_activity(self, actor, verb, obj, target=None,
+            related_nodes=None):
+        from allura.model.project import Project
+        super(Director, self).create_activity(actor, verb, obj,
+                target=target, related_nodes=related_nodes)
+        # aggregate actor and follower's timelines
+        create_timelines.post(actor.node_id)
+        # aggregate project and follower's timelines
+        for node in [obj, target] + (related_nodes or []):
+            if isinstance(node, Project):
+                create_timelines.post(node.node_id)
+
+
+class Aggregator(BaseAggregator):
+    pass
 
 
-class ActivityNode(base.NodeBase):
+class ActivityNode(NodeBase):
     @property
     def node_id(self):
         return "%s:%s" % (self.__class__.__name__, self._id)
 
 
-class ActivityObject(base.ActivityObjectBase):
+class ActivityObject(ActivityObjectBase):
     @property
     def activity_name(self):
         """Override this for each Artifact type."""

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dca06665/Allura/allura/tasks/activity_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/activity_tasks.py b/Allura/allura/tasks/activity_tasks.py
new file mode 100644
index 0000000..26ac02a
--- /dev/null
+++ b/Allura/allura/tasks/activity_tasks.py
@@ -0,0 +1,25 @@
+#       Licensed to the Apache Software Foundation (ASF) under one
+#       or more contributor license agreements.  See the NOTICE file
+#       distributed with this work for additional information
+#       regarding copyright ownership.  The ASF licenses this file
+#       to you under the Apache License, Version 2.0 (the
+#       "License"); you may not use this file except in compliance
+#       with the License.  You may obtain a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#       Unless required by applicable law or agreed to in writing,
+#       software distributed under the License is distributed on an
+#       "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+#       KIND, either express or implied.  See the License for the
+#       specific language governing permissions and limitations
+#       under the License.
+
+from pylons import app_globals as g
+
+from allura.lib.decorators import task
+
+
+@task
+def create_timelines(node_id):
+    g.director.create_timelines(node_id)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dca06665/Allura/setup.py
----------------------------------------------------------------------
diff --git a/Allura/setup.py b/Allura/setup.py
index bc829ce..4f3cf84 100644
--- a/Allura/setup.py
+++ b/Allura/setup.py
@@ -146,7 +146,9 @@ setup(
     [easy_widgets.engines]
     jinja = allura.config.app_cfg:JinjaEngine
 
-    [activitystream.storage]
-    driver = activitystream.storage.mingstorage:MingStorage
+    [activitystream]
+    storage = activitystream.storage.mingstorage:MingStorage
+    director = allura.model.timeline:Director
+    aggregator = allura.model.timeline:Aggregator
     """,
 )

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/dca06665/ForgeActivity/forgeactivity/tests/functional/test_root.py
----------------------------------------------------------------------
diff --git a/ForgeActivity/forgeactivity/tests/functional/test_root.py b/ForgeActivity/forgeactivity/tests/functional/test_root.py
index 178bb89..eda71be 100644
--- a/ForgeActivity/forgeactivity/tests/functional/test_root.py
+++ b/ForgeActivity/forgeactivity/tests/functional/test_root.py
@@ -21,7 +21,9 @@ from tg import config
 
 import dateutil.parser
 from nose.tools import assert_equal
+from pylons import app_globals as g
 
+from allura import model as M
 from alluratest.controller import TestController
 from allura.tests import decorators as td
 
@@ -148,3 +150,23 @@ class TestActivityController(TestController):
         assert director.get_timeline.call_count == 1
         assert director.get_timeline.call_args[0][0].shortname == 'test'
         assert director.get_timeline.call_args[1]['actor_only'] == False
+
+    @td.with_tracker
+    @td.with_tool('u/test-user-1', 'activity')
+    @td.with_user_project('test-user-1')
+    def test_background_aggregation(self):
+        self.app.get('/u/test-admin/activity/follow?follow=True',
+                extra_environ=dict(username='test-user-1'))
+        # new ticket, creates activity
+        d = {'ticket_form.summary': 'New Ticket'}
+        self.app.post('/bugs/save_ticket', params=d)
+        orig_create_timeline = g.director.aggregator.create_timeline
+        with patch.object(g.director.aggregator, 'create_timeline') as create_timeline:
+            create_timeline.side_effect = orig_create_timeline
+            M.MonQTask.run_ready()
+            # 3 aggregations: 1 actor, 1 follower, 1 project
+            assert_equal(create_timeline.call_count, 3)
+            create_timeline.reset_mock()
+            self.app.get('/u/test-admin/activity/')
+            self.app.get('/u/test-user-1/activity/')
+            assert_equal(create_timeline.call_count, 0)


Mime
View raw message