incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject [3/3] git commit: [#5023] add indexed queue_empty field so mailbox queries aren't slow on large collections
Date Wed, 03 Oct 2012 20:50:09 GMT
[#5023] add indexed queue_empty field so mailbox queries aren't slow on large collections


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/6f52980c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/6f52980c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/6f52980c

Branch: refs/heads/master
Commit: 6f52980c9c0026813e4a8ac5680e146b63e4cb4d
Parents: e846983
Author: Dave Brondsema <dbrondsema@geek.net>
Authored: Wed Oct 3 18:44:01 2012 +0000
Committer: Dave Brondsema <dbrondsema@geek.net>
Committed: Wed Oct 3 18:44:01 2012 +0000

----------------------------------------------------------------------
 Allura/allura/lib/helpers.py                   |    1 +
 Allura/allura/model/notification.py            |   22 ++++++++++++++----
 Allura/allura/tests/model/test_notification.py |    6 +++++
 3 files changed, 24 insertions(+), 5 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/6f52980c/Allura/allura/lib/helpers.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/helpers.py b/Allura/allura/lib/helpers.py
index a48822c..4cf3fbc 100644
--- a/Allura/allura/lib/helpers.py
+++ b/Allura/allura/lib/helpers.py
@@ -425,6 +425,7 @@ def pop_user_notifications(user=None):
     if mbox:
         notifications = M.Notification.query.find(dict(_id={'$in':mbox.queue}))
         mbox.queue = []
+        mbox.queue_empty = True
         for n in notifications:
             M.Notification.query.remove({'_id': n._id}) # clean it up so it doesn't hang
around
             yield n

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/6f52980c/Allura/allura/model/notification.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/notification.py b/Allura/allura/model/notification.py
index 9bd9180..ff761a3 100644
--- a/Allura/allura/model/notification.py
+++ b/Allura/allura/model/notification.py
@@ -103,6 +103,7 @@ class Notification(MappedClass):
         n = cls._make_notification(artifact, topic, **kw)
         if n:
             mbox.queue.append(n._id)
+            mbox.queue_empty = False
         return n
 
     @classmethod
@@ -291,7 +292,9 @@ class Mailbox(MappedClass):
         indexes = [
             ('project_id', 'artifact_index_id'),
             ('is_flash', 'user_id'),
-            ('type', 'next_scheduled')]
+            ('type', 'next_scheduled'),  # for q_digest
+            ('type', 'queue_empty'),  # for q_direct
+        ]
 
     _id = FieldProperty(S.ObjectId)
     user_id = ForeignIdProperty('User', if_missing=lambda:c.user._id)
@@ -314,6 +317,7 @@ class Mailbox(MappedClass):
 
     # a list of notification _id values
     queue = FieldProperty([str])
+    queue_empty = FieldProperty(bool)
 
     project = RelationProperty('Project')
     app_config = RelationProperty('AppConfig')
@@ -423,7 +427,9 @@ class Mailbox(MappedClass):
         for mbox in cls.query.find(d):
             mbox.query.update(
                 {'$push':dict(queue=nid),
-                 '$set':dict(last_modified=datetime.utcnow())})
+                 '$set':dict(last_modified=datetime.utcnow(),
+                             queue_empty=False),
+                })
             # Make sure the mbox doesn't stick around to be flush()ed
             session(mbox).expunge(mbox)
 
@@ -437,7 +443,9 @@ class Mailbox(MappedClass):
         # Queries to find all matching subscription objects
         q_direct = dict(
             type='direct',
-            queue={'$ne':[]})
+            queue={'$ne':[]},
+            queue_empty=False,
+        )
         if MAILBOX_QUIESCENT:
             q_direct['last_modified']={'$lt':now - MAILBOX_QUIESCENT}
         q_digest = dict(
@@ -447,7 +455,9 @@ class Mailbox(MappedClass):
             mbox = cls.query.find_and_modify(
                 query=dict(_id=mbox._id),
                 update={'$set': dict(
-                        queue=[])},
+                            queue=[],
+                            queue_empty=True,
+                        )},
                 new=False)
             mbox.fire(now)
         for mbox in cls.query.find(q_digest):
@@ -462,7 +472,9 @@ class Mailbox(MappedClass):
                 query=dict(_id=mbox._id),
                 update={'$set': dict(
                         next_scheduled=next_scheduled,
-                        queue=[])},
+                        queue=[],
+                        queue_empty=True,
+                        )},
                 new=False)
             mbox.fire(now)
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/6f52980c/Allura/allura/tests/model/test_notification.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/model/test_notification.py b/Allura/allura/tests/model/test_notification.py
index 4e9a969..d59fc1c 100644
--- a/Allura/allura/tests/model/test_notification.py
+++ b/Allura/allura/tests/model/test_notification.py
@@ -96,6 +96,7 @@ class TestPostNotifications(unittest.TestCase):
         assert M.Mailbox.query.find().count()==1
         mbox = M.Mailbox.query.get()
         assert len(mbox.queue) == 1
+        assert not mbox.queue_empty
 
     def test_email(self):
         self._subscribe()  # as current user: test-admin
@@ -111,7 +112,9 @@ class TestPostNotifications(unittest.TestCase):
         mboxes = M.Mailbox.query.find().all()
         assert_equal(len(mboxes), 2)
         assert_equal(len(mboxes[0].queue), 1)
+        assert not mboxes[0].queue_empty
         assert_equal(len(mboxes[1].queue), 1)
+        assert not mboxes[1].queue_empty
 
         M.Mailbox.fire_ready()
         email_tasks = M.MonQTask.query.find({'state': 'ready'}).all()
@@ -200,10 +203,13 @@ class TestSubscriptionTypes(unittest.TestCase):
 
     def test_message(self):
         self._test_message()
+        
         self.setUp()
         self._test_message()
+
         self.setUp()
         M.notification.MAILBOX_QUIESCENT=timedelta(minutes=1)
+        # will raise "assert msg is not None" since the new message is not 1 min old:
         self.assertRaises(AssertionError, self._test_message)
 
     def _test_message(self):


Mime
View raw message