incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From john...@apache.org
Subject [26/34] git commit: [#5265] Control Solr commit options from config file
Date Wed, 05 Dec 2012 17:27:49 GMT
[#5265] Control Solr commit options from config file


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/9e08c7d0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/9e08c7d0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/9e08c7d0

Branch: refs/heads/cj/4691
Commit: 9e08c7d0abfe79dc01ed3a0efe992f2114fc6efd
Parents: 8c8ad4d
Author: Tim Van Steenburgh <tvansteenburgh@gmail.com>
Authored: Tue Nov 20 21:12:27 2012 +0000
Committer: Cory Johns <johnsca@geek.net>
Committed: Mon Dec 3 22:13:24 2012 +0000

----------------------------------------------------------------------
 Allura/allura/lib/app_globals.py      |   73 ++--------------------
 Allura/allura/lib/solr.py             |   91 ++++++++++++++++++++++++++++
 Allura/allura/tests/unit/test_solr.py |   30 +++++++++
 Allura/development.ini                |    4 +
 4 files changed, 131 insertions(+), 67 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9e08c7d0/Allura/allura/lib/app_globals.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 1789855..a263177 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -6,14 +6,12 @@ __all__ = ['Globals']
 import logging
 import cgi
 import json
-import shlex
 import datetime
 from urllib import urlencode
 from subprocess import Popen, PIPE
 
 import activitystream
 import pkg_resources
-import pysolr
 import markdown
 import pygments
 import pygments.lexers
@@ -38,21 +36,24 @@ from allura.lib import helpers as h
 from allura.lib.widgets import analytics
 from allura.lib.security import Credentials
 from allura.lib.async import Connection, MockAMQ
+from allura.lib.solr import Solr, MockSOLR
 from allura.lib.zarkov_helpers import ZarkovClient, zmq
 
 log = logging.getLogger(__name__)
 
+
 class ForgeMarkdown(markdown.Markdown):
     def convert(self, source):
         try:
             return markdown.Markdown.convert(self, source)
-        except Exception as e:
+        except Exception:
             log.info('Invalid markdown: %s', source, exc_info=True)
             escaped = h.really_unicode(source)
             escaped = cgi.escape(escaped)
             return h.html.literal(u"""<p><strong>ERROR!</strong> The markdown
supplied could not be parsed correctly.
             Did you forget to surround a code snippet with "~~~~"?</p><pre>%s</pre>"""
% escaped)
 
+
 class Globals(object):
     """Container for objects available throughout the life of the application.
 
@@ -72,7 +73,8 @@ class Globals(object):
         if asbool(config.get('solr.mock')):
             self.solr = MockSOLR()
         elif self.solr_server:
-            self.solr = pysolr.Solr(self.solr_server)
+            self.solr = Solr(self.solr_server, commit=asbool(config.get('solr.commit', True)),
+                    commitWithin=config.get('solr.commitWithin'))
         else: # pragma no cover
             self.solr = None
         self.use_queue = asbool(config.get('use_queue', False))
@@ -422,69 +424,6 @@ class Globals(object):
     def year(self):
         return datetime.datetime.utcnow().year
 
-class MockSOLR(object):
-
-    class MockHits(list):
-        @property
-        def hits(self):
-            return len(self)
-
-        @property
-        def docs(self):
-            return self
-
-    def __init__(self):
-        self.db = {}
-
-    def add(self, objects):
-        for o in objects:
-            o['text'] = ''.join(o['text'])
-            self.db[o['id']] = o
-
-    def commit(self):
-        pass
-
-    def search(self, q, fq=None, **kw):
-        if isinstance(q, unicode):
-            q = q.encode('latin-1')
-        # Parse query
-        preds = []
-        q_parts = shlex.split(q)
-        if fq: q_parts += fq
-        for part in q_parts:
-            if part == '&&':
-                continue
-            if ':' in part:
-                field, value = part.split(':', 1)
-                preds.append((field, value))
-            else:
-                preds.append(('text', part))
-        result = self.MockHits()
-        for obj in self.db.values():
-            for field, value in preds:
-                neg = False
-                if field[0] == '!':
-                    neg = True
-                    field = field[1:]
-                if field == 'text' or field.endswith('_t'):
-                    if (value not in str(obj.get(field, ''))) ^ neg:
-                        break
-                else:
-                    if (value != str(obj.get(field, ''))) ^ neg:
-                        break
-            else:
-                result.append(obj)
-        return result
-
-    def delete(self, *args, **kwargs):
-        if kwargs.get('q', None) == '*:*':
-            self.db = {}
-        elif kwargs.get('id', None):
-            del self.db[kwargs['id']]
-        elif kwargs.get('q', None):
-            for doc in self.search(kwargs['q']):
-                self.delete(id=doc['id'])
-
 class Icon(object):
     def __init__(self, char, css):
         self.char = char

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9e08c7d0/Allura/allura/lib/solr.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py
new file mode 100644
index 0000000..99af9f1
--- /dev/null
+++ b/Allura/allura/lib/solr.py
@@ -0,0 +1,91 @@
+import shlex
+import pysolr
+
+
+class Solr(pysolr.Solr):
+    """Solr server that accepts default values for `commit` and
+    `commitWithin` and passes those values through to each `add` and
+    `delete` call, unless explicitly overridden.
+    """
+
+    def __init__(self, server, commit=True, commitWithin=None):
+        pysolr.Solr.__init__(self, server)
+        self.commit = commit
+        self.commitWithin = commitWithin
+
+    def add(self, *args, **kw):
+        if 'commit' not in kw:
+            kw['commit'] = self.commit
+        if self.commitWithin and 'commitWithin' not in kw:
+            kw['commitWithin'] = self.commitWithin
+        return pysolr.Solr.add(self, *args, **kw)
+
+    def delete(self, *args, **kw):
+        if 'commit' not in kw:
+            kw['commit'] = self.commit
+        return pysolr.Solr.delete(self, *args, **kw)
+
+
+class MockSOLR(object):
+
+    class MockHits(list):
+        @property
+        def hits(self):
+            return len(self)
+
+        @property
+        def docs(self):
+            return self
+
+    def __init__(self):
+        self.db = {}
+
+    def add(self, objects):
+        for o in objects:
+            o['text'] = ''.join(o['text'])
+            self.db[o['id']] = o
+
+    def commit(self):
+        pass
+
+    def search(self, q, fq=None, **kw):
+        if isinstance(q, unicode):
+            q = q.encode('latin-1')
+        # Parse query
+        preds = []
+        q_parts = shlex.split(q)
+        if fq: q_parts += fq
+        for part in q_parts:
+            if part == '&&':
+                continue
+            if ':' in part:
+                field, value = part.split(':', 1)
+                preds.append((field, value))
+            else:
+                preds.append(('text', part))
+        result = self.MockHits()
+        for obj in self.db.values():
+            for field, value in preds:
+                neg = False
+                if field[0] == '!':
+                    neg = True
+                    field = field[1:]
+                if field == 'text' or field.endswith('_t'):
+                    if (value not in str(obj.get(field, ''))) ^ neg:
+                        break
+                else:
+                    if (value != str(obj.get(field, ''))) ^ neg:
+                        break
+            else:
+                result.append(obj)
+        return result
+
+    def delete(self, *args, **kwargs):
+        if kwargs.get('q', None) == '*:*':
+            self.db = {}
+        elif kwargs.get('id', None):
+            del self.db[kwargs['id']]
+        elif kwargs.get('q', None):
+            for doc in self.search(kwargs['q']):
+                self.delete(id=doc['id'])
+

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9e08c7d0/Allura/allura/tests/unit/test_solr.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/unit/test_solr.py b/Allura/allura/tests/unit/test_solr.py
new file mode 100644
index 0000000..2e5b668
--- /dev/null
+++ b/Allura/allura/tests/unit/test_solr.py
@@ -0,0 +1,30 @@
+import unittest
+import mock
+
+from allura.lib.solr import Solr
+
+class TestSolr(unittest.TestCase):
+    @mock.patch('allura.lib.solr.pysolr')
+    def setUp(self, pysolr):
+        self.solr = Solr('server', commit=False, commitWithin='10000')
+
+    @mock.patch('allura.lib.solr.pysolr')
+    def test_add(self, pysolr):
+        s = self.solr
+        s.add('foo', commit=True, commitWithin=None)
+        pysolr.Solr.add.assert_called_once_with(s, 'foo', commit=True,
+                commitWithin=None)
+        pysolr.reset_mock()
+        s.add('bar', somekw='value')
+        pysolr.Solr.add.assert_called_once_with(s, 'bar', commit=False,
+                commitWithin='10000', somekw='value')
+
+    @mock.patch('allura.lib.solr.pysolr')
+    def test_delete(self, pysolr):
+        s = self.solr
+        s.delete('foo', commit=True)
+        pysolr.Solr.delete.assert_called_once_with(s, 'foo', commit=True)
+        pysolr.reset_mock()
+        s.delete('bar', somekw='value')
+        pysolr.Solr.delete.assert_called_once_with(s, 'bar', commit=False,
+                somekw='value')

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/9e08c7d0/Allura/development.ini
----------------------------------------------------------------------
diff --git a/Allura/development.ini b/Allura/development.ini
index 4ba35d7..7ac4897 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -157,6 +157,10 @@ openid.realm = http://localhost:8080/
 
 # SOLR setup
 solr.server = http://localhost:8983/solr/core0
+# commit on every add/delete?
+solr.commit = false
+# commit add operations within N ms
+solr.commitWithin = 10000
 
 # Forgemail server
 forgemail.host = 0.0.0.0


Mime
View raw message