incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject [28/50] git commit: [#6735] Add Markdown caching to Post artifacts
Date Tue, 22 Oct 2013 21:50:38 GMT
[#6735] Add Markdown caching to Post artifacts

Signed-off-by: Tim Van Steenburgh <tvansteenburgh@gmail.com>


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/c58230fe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/c58230fe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/c58230fe

Branch: refs/heads/cj/6422
Commit: c58230fee5223f58c6aac2590094c72ae7ea7b39
Parents: 7b02284
Author: Tim Van Steenburgh <tvansteenburgh@gmail.com>
Authored: Wed Oct 9 16:42:21 2013 +0000
Committer: Cory Johns <cjohns@slashdotmedia.com>
Committed: Tue Oct 15 21:29:04 2013 +0000

----------------------------------------------------------------------
 Allura/allura/lib/app_globals.py                | 35 ++++++++++
 Allura/allura/model/__init__.py                 |  2 +-
 Allura/allura/model/discuss.py                  |  3 +-
 Allura/allura/model/types.py                    |  9 +++
 .../allura/templates/widgets/post_widget.html   |  2 +-
 Allura/allura/tests/test_globals.py             | 71 ++++++++++++++++++++
 Allura/development.ini                          |  6 ++
 7 files changed, 125 insertions(+), 3 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c58230fe/Allura/allura/lib/app_globals.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 0da0eee..083083a 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -23,11 +23,13 @@
 __all__ = ['Globals']
 import logging
 import cgi
+import hashlib
 import json
 import datetime
 from urllib import urlencode
 from subprocess import Popen, PIPE
 import os
+import time
 import traceback
 
 import activitystream
@@ -80,6 +82,39 @@ class ForgeMarkdown(markdown.Markdown):
             return h.html.literal(u"""<p><strong>ERROR!</strong> The markdown
supplied could not be parsed correctly.
             Did you forget to surround a code snippet with "~~~~"?</p><pre>%s</pre>"""
% escaped)
 
+    def cached_convert(self, artifact, field_name):
+        """Convert ``artifact.field_name`` markdown source to html, caching
+        the result if the render time is greater than the defined threshhold.
+
+        """
+        source_text = getattr(artifact, field_name)
+        cache_field_name = field_name + '_cache'
+        cache = getattr(artifact, cache_field_name, None)
+        if not cache:
+            log.warn('Skipping Markdown caching - Missing cache field "%s" on class %s',
+                    field_name, artifact.__class__.__name__)
+            return self.convert(source_text)
+
+        md5 = hashlib.md5(source_text).hexdigest()
+        if cache.md5 == md5:
+            return cache.html
+
+        start = time.time()
+        html = self.convert(source_text)
+        render_time = time.time() - start
+
+        threshhold = config.get('markdown_cache_threshhold')
+        try:
+            threshhold = float(threshhold) if threshhold else None
+        except ValueError:
+            threshhold = None
+            log.warn('Skipping Markdown caching - The value for config param '
+                    '"markdown_cache_threshhold" must be a float.')
+
+        if threshhold != None and render_time > threshhold:
+            cache.md5, cache.html, cache.render_time = md5, html, render_time
+        return html
+
 
 class Globals(object):
     """Container for objects available throughout the life of the application.

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c58230fe/Allura/allura/model/__init__.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/__init__.py b/Allura/allura/model/__init__.py
index 27055c8..5c295d1 100644
--- a/Allura/allura/model/__init__.py
+++ b/Allura/allura/model/__init__.py
@@ -36,7 +36,7 @@ from .stats import Stats
 from .oauth import OAuthToken, OAuthConsumerToken, OAuthRequestToken, OAuthAccessToken
 from .monq_model import MonQTask
 
-from .types import ACE, ACL, EVERYONE, ALL_PERMISSIONS, DENY_ALL
+from .types import ACE, ACL, EVERYONE, ALL_PERMISSIONS, DENY_ALL, MarkdownCache
 from .session import main_doc_session, main_orm_session
 from .session import project_doc_session, project_orm_session
 from .session import artifact_orm_session, repository_orm_session

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c58230fe/Allura/allura/model/discuss.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/discuss.py b/Allura/allura/model/discuss.py
index a8cd7a3..394ad2e 100644
--- a/Allura/allura/model/discuss.py
+++ b/Allura/allura/model/discuss.py
@@ -33,11 +33,11 @@ from allura.lib import security
 from allura.lib.security import require_access, has_access
 from allura.lib import utils
 from allura.model.notification import Notification, Mailbox
-from allura.model.auth import ProjectRole
 from .artifact import Artifact, ArtifactReference, VersionedArtifact, Snapshot, Message,
Feed
 from .attachments import BaseAttachment
 from .auth import User
 from .timeline import ActivityObject
+from .types import MarkdownCache
 
 log = logging.getLogger(__name__)
 
@@ -452,6 +452,7 @@ class Post(Message, VersionedArtifact, ActivityObject):
     last_edit_by_id = ForeignIdProperty(User)
     edit_count = FieldProperty(int, if_missing=0)
     spam_check_id = FieldProperty(str, if_missing='')
+    text_cache = FieldProperty(MarkdownCache)
 
     thread = RelationProperty(Thread)
     discussion = RelationProperty(Discussion)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c58230fe/Allura/allura/model/types.py
----------------------------------------------------------------------
diff --git a/Allura/allura/model/types.py b/Allura/allura/model/types.py
index 2e14c86..ab7d341 100644
--- a/Allura/allura/model/types.py
+++ b/Allura/allura/model/types.py
@@ -20,6 +20,15 @@ from ming import schema as S
 
 EVERYONE, ALL_PERMISSIONS = None, '*'
 
+class MarkdownCache(S.Object):
+    def __init__(self, **kw):
+        super(MarkdownCache, self).__init__(
+            fields=dict(
+                md5=S.String(),
+                html=S.String(),
+                render_time=S.Float()),
+            **kw)
+
 class ACE(S.Object):
     '''ACE - access control entry'''
     ALLOW, DENY = 'ALLOW', 'DENY'

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c58230fe/Allura/allura/templates/widgets/post_widget.html
----------------------------------------------------------------------
diff --git a/Allura/allura/templates/widgets/post_widget.html b/Allura/allura/templates/widgets/post_widget.html
index 6c33f62..5cc876b 100644
--- a/Allura/allura/templates/widgets/post_widget.html
+++ b/Allura/allura/templates/widgets/post_widget.html
@@ -47,7 +47,7 @@
             {% if show_subject %}
                 <b>{{value.subject or '(no subject)'}}<br/></b>
             {% endif %}
-            {{g.markdown.convert(value.text)|safe}}&nbsp;
+            {{g.markdown.cached_convert(value, 'text')|safe}}&nbsp;
             {{lib.related_artifacts(value)}}
             {% if value.edit_count %}
                 <br><small>Last edit: {{value.last_edit_by().display_name}} {{h.ago(value.last_edit_date)}}</small>

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c58230fe/Allura/allura/tests/test_globals.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_globals.py b/Allura/allura/tests/test_globals.py
index 3ce26bb..f6d6849 100644
--- a/Allura/allura/tests/test_globals.py
+++ b/Allura/allura/tests/test_globals.py
@@ -20,6 +20,8 @@
 
 import re
 import os, allura
+import unittest
+import hashlib
 from mock import patch
 from urllib import quote
 
@@ -33,6 +35,7 @@ from alluratest.controller import setup_basic_test, setup_global_objects
 
 from allura import model as M
 from allura.lib import helpers as h
+from allura.lib.app_globals import ForgeMarkdown
 from allura.tests import decorators as td
 
 from forgewiki import model as WM
@@ -542,3 +545,71 @@ def get_projects_property_in_the_same_order(names, prop):
     projects = M.Project.query.find(dict(name={'$in': names})).all()
     projects_dict = dict([(p['name'],p[prop]) for p in projects])
     return [projects_dict[name] for name in names]
+
+
+class TestCachedMarkdown(unittest.TestCase):
+    def setUp(self):
+        self.md = ForgeMarkdown()
+        self.post = M.Post()
+        self.post.text = u'**bold**'
+        self.expected_html = u'<p><strong>bold</strong></p>'
+
+    def test_bad_source_field_name(self):
+        self.assertRaises(AttributeError, self.md.cached_convert, self.post, 'no_such_field')
+
+    def test_missing_cache_field(self):
+        delattr(self.post, 'text_cache')
+        html = self.md.cached_convert(self.post, 'text')
+        self.assertEqual(html, self.expected_html)
+
+    @patch.dict('allura.lib.app_globals.config', markdown_cache_threshhold='0')
+    def test_empty_cache(self):
+        html = self.md.cached_convert(self.post, 'text')
+        self.assertEqual(html, self.expected_html)
+        self.assertEqual(html, self.post.text_cache.html)
+        self.assertEqual(hashlib.md5(self.post.text).hexdigest(),
+                self.post.text_cache.md5)
+        self.assertTrue(self.post.text_cache.render_time > 0)
+
+    @patch.dict('allura.lib.app_globals.config', markdown_cache_threshhold='0')
+    def test_stale_cache(self):
+        old = self.md.cached_convert(self.post, 'text')
+        self.post.text = u'new, different source text'
+        html = self.md.cached_convert(self.post, 'text')
+        self.assertNotEqual(old, html)
+        self.assertEqual(html, self.post.text_cache.html)
+        self.assertEqual(hashlib.md5(self.post.text).hexdigest(),
+                self.post.text_cache.md5)
+        self.assertTrue(self.post.text_cache.render_time > 0)
+
+    @patch.dict('allura.lib.app_globals.config', markdown_cache_threshhold='0')
+    def test_valid_cache(self):
+        self.md.cached_convert(self.post, 'text')
+        with patch.object(self.md, 'convert') as convert_func:
+            html = self.md.cached_convert(self.post, 'text')
+            self.assertEqual(html, self.expected_html)
+            self.assertFalse(convert_func.called)
+
+    @patch.dict('allura.lib.app_globals.config', {})
+    def test_no_threshhold_defined(self):
+        html = self.md.cached_convert(self.post, 'text')
+        self.assertEqual(html, self.expected_html)
+        self.assertIsNone(self.post.text_cache.md5)
+        self.assertIsNone(self.post.text_cache.html)
+        self.assertIsNone(self.post.text_cache.render_time)
+
+    @patch.dict('allura.lib.app_globals.config', markdown_cache_threshhold='foo')
+    def test_invalid_threshhold(self):
+        html = self.md.cached_convert(self.post, 'text')
+        self.assertEqual(html, self.expected_html)
+        self.assertIsNone(self.post.text_cache.md5)
+        self.assertIsNone(self.post.text_cache.html)
+        self.assertIsNone(self.post.text_cache.render_time)
+
+    @patch.dict('allura.lib.app_globals.config', markdown_cache_threshhold='99999')
+    def test_render_time_below_threshhold(self):
+        html = self.md.cached_convert(self.post, 'text')
+        self.assertEqual(html, self.expected_html)
+        self.assertIsNone(self.post.text_cache.md5)
+        self.assertIsNone(self.post.text_cache.html)
+        self.assertIsNone(self.post.text_cache.render_time)

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/c58230fe/Allura/development.ini
----------------------------------------------------------------------
diff --git a/Allura/development.ini b/Allura/development.ini
index cff48fd..241e491 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -299,6 +299,12 @@ forgemail.url = http://localhost:8080
 auth.method = local
 registration.method = local
 
+# When rendering discussion post Markdown to html, if the render takes longer
+# than `markdown_cache_threshhold` (in seconds), the resulting html will be
+# cached and served from cache on subsequent requests. Set to 0 to cache all
+# posts. Remove entirely to cache nothing.
+markdown_cache_threshhold = .1
+
 [app:task]
 use = main
 override_root = task


Mime
View raw message