incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tvansteenbu...@apache.org
Subject [29/50] [abbrv] git commit: [#6161] ticket:329 Push changes to multiple solr instances
Date Tue, 07 May 2013 22:29:03 GMT
[#6161] ticket:329 Push changes to multiple solr instances


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/7a332c6e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/7a332c6e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/7a332c6e

Branch: refs/heads/tv/4321
Commit: 7a332c6ecdc6c2cffd5f262d0293d7afdf195d85
Parents: ba100cc
Author: Igor Bondarenko <jetmind2@gmail.com>
Authored: Mon Apr 29 12:46:33 2013 +0000
Committer: Dave Brondsema <dbrondsema@slashdotmedia.com>
Committed: Thu May 2 22:18:14 2013 +0000

----------------------------------------------------------------------
 Allura/allura/lib/app_globals.py      |   19 +++++---
 Allura/allura/lib/solr.py             |   47 +++++++++++++++-----
 Allura/allura/tests/unit/test_solr.py |   67 +++++++++++++++++++++------
 Allura/development.ini                |    3 +
 4 files changed, 104 insertions(+), 32 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/7a332c6e/Allura/allura/lib/app_globals.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 0621b8d..b030d6f 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -38,7 +38,7 @@ import pygments.util
 from tg import config, session
 from pylons import request
 from pylons import tmpl_context as c
-from paste.deploy.converters import asbool, asint
+from paste.deploy.converters import asbool, asint, aslist
 from pypeline.markup import markup as pypeline_markup
 
 import ew as ew_core
@@ -93,14 +93,21 @@ class Globals(object):
         self.allura_templates = pkg_resources.resource_filename('allura', 'templates')
 
         # Setup SOLR
-        self.solr_server = config.get('solr.server')
+        self.solr_server = aslist(config.get('solr.server'), ',')
+        # skip empty strings in case of extra commas
+        self.solr_server = [s for s in self.solr_server if s]
+        self.solr_query_server = config.get('solr.query_server')
         if asbool(config.get('solr.mock')):
             self.solr = self.solr_short_timeout = MockSOLR()
         elif self.solr_server:
-            self.solr = Solr(self.solr_server, commit=asbool(config.get('solr.commit', True)),
-                    commitWithin=config.get('solr.commitWithin'), timeout=int(config.get('solr.long_timeout',
60)))
-            self.solr_short_timeout = Solr(self.solr_server, commit=asbool(config.get('solr.commit',
True)),
-                    commitWithin=config.get('solr.commitWithin'), timeout=int(config.get('solr.short_timeout',
10)))
+            self.solr = Solr(self.solr_server, self.solr_query_server,
+                             commit=asbool(config.get('solr.commit', True)),
+                             commitWithin=config.get('solr.commitWithin'),
+                             timeout=int(config.get('solr.long_timeout', 60)))
+            self.solr_short_timeout = Solr(self.solr_server, self.solr_query_server,
+                                           commit=asbool(config.get('solr.commit', True)),
+                                           commitWithin=config.get('solr.commitWithin'),
+                                           timeout=int(config.get('solr.short_timeout', 10)))
         else: # pragma no cover
             self.solr = None
             self.solr_short_timeout = None

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/7a332c6e/Allura/allura/lib/solr.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py
index e91b183..e1e9567 100644
--- a/Allura/allura/lib/solr.py
+++ b/Allura/allura/lib/solr.py
@@ -20,28 +20,53 @@ import pysolr
 from pysolr import SolrError
 
 
-class Solr(pysolr.Solr):
-    """Solr server that accepts default values for `commit` and
-    `commitWithin` and passes those values through to each `add` and
-    `delete` call, unless explicitly overridden.
+class Solr(object):
+    """Solr interface that pushes updates to multiple solr instances.
+
+    `push_servers`: list of servers to push to.
+    `query_server`: server to read from. Uses `push_servers[0]` if not specified.
+
+    Also, accepts default values for `commit` and `commitWithin`
+    and passes those values through to each `add` and `delete` call,
+    unless explicitly overridden.
     """
 
-    def __init__(self, server, commit=True, commitWithin=None, **kw):
-        pysolr.Solr.__init__(self, server, **kw)
-        self.commit = commit
+    def __init__(self, push_servers, query_server=None,
+                 commit=True, commitWithin=None, **kw):
+        self.push_pool = [pysolr.Solr(s, **kw) for s in push_servers]
+        if query_server:
+            self.query_server = pysolr.Solr(query_server, **kw)
+        else:
+            self.query_server = self.push_pool[0]
+        self._commit = commit
         self.commitWithin = commitWithin
 
     def add(self, *args, **kw):
         if 'commit' not in kw:
-            kw['commit'] = self.commit
+            kw['commit'] = self._commit
         if self.commitWithin and 'commitWithin' not in kw:
             kw['commitWithin'] = self.commitWithin
-        return pysolr.Solr.add(self, *args, **kw)
+        responses = []
+        for solr in self.push_pool:
+            responses.append(solr.add(*args, **kw))
+        return responses
 
     def delete(self, *args, **kw):
         if 'commit' not in kw:
-            kw['commit'] = self.commit
-        return pysolr.Solr.delete(self, *args, **kw)
+            kw['commit'] = self._commit
+        responses = []
+        for solr in self.push_pool:
+            responses.append(solr.delete(*args, **kw))
+        return responses
+
+    def commit(self, *args, **kw):
+        responses = []
+        for solr in self.push_pool:
+            responses.append(solr.commit(*args, **kw))
+        return responses
+
+    def search(self, *args, **kw):
+        return self.query_server.search(*args, **kw)
 
 
 class MockSOLR(object):

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/7a332c6e/Allura/allura/tests/unit/test_solr.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/unit/test_solr.py b/Allura/allura/tests/unit/test_solr.py
index b3dee43..2bc5cdb 100644
--- a/Allura/allura/tests/unit/test_solr.py
+++ b/Allura/allura/tests/unit/test_solr.py
@@ -28,30 +28,67 @@ from allura.lib.solr import Solr
 from allura.lib.search import solarize, search_app
 
 class TestSolr(unittest.TestCase):
+
     @mock.patch('allura.lib.solr.pysolr')
-    def setUp(self, pysolr):
-        self.solr = Solr('server', commit=False, commitWithin='10000')
+    def test_init(self, pysolr):
+        servers = ['server1', 'server2']
+        solr = Solr(servers, commit=False, commitWithin='10000')
+        calls = [mock.call('server1'), mock.call('server2')]
+        pysolr.Solr.assert_has_calls(calls)
+        assert_equal(len(solr.push_pool), 2)
+
+        pysolr.reset_mock()
+        solr = Solr(servers, 'server3', commit=False, commitWithin='10000')
+        calls = [mock.call('server1'), mock.call('server2'), mock.call('server3')]
+        pysolr.Solr.assert_has_calls(calls)
+        assert_equal(len(solr.push_pool), 2)
 
     @mock.patch('allura.lib.solr.pysolr')
     def test_add(self, pysolr):
-        s = self.solr
-        s.add('foo', commit=True, commitWithin=None)
-        pysolr.Solr.add.assert_called_once_with(s, 'foo', commit=True,
-                commitWithin=None)
+        servers = ['server1', 'server2']
+        solr = Solr(servers, commit=False, commitWithin='10000')
+        solr.add('foo', commit=True, commitWithin=None)
+        calls = [mock.call('foo', commit=True, commitWithin=None)] * 2
+        pysolr.Solr().add.assert_has_calls(calls)
         pysolr.reset_mock()
-        s.add('bar', somekw='value')
-        pysolr.Solr.add.assert_called_once_with(s, 'bar', commit=False,
-                commitWithin='10000', somekw='value')
+        solr.add('bar', somekw='value')
+        calls = [mock.call('bar', commit=False,
+            commitWithin='10000', somekw='value')] * 2
+        pysolr.Solr().add.assert_has_calls(calls)
 
     @mock.patch('allura.lib.solr.pysolr')
     def test_delete(self, pysolr):
-        s = self.solr
-        s.delete('foo', commit=True)
-        pysolr.Solr.delete.assert_called_once_with(s, 'foo', commit=True)
+        servers = ['server1', 'server2']
+        solr = Solr(servers, commit=False, commitWithin='10000')
+        solr.delete('foo', commit=True)
+        calls = [mock.call('foo', commit=True)] * 2
+        pysolr.Solr().delete.assert_has_calls(calls)
+        pysolr.reset_mock()
+        solr.delete('bar', somekw='value')
+        calls = [mock.call('bar', commit=False, somekw='value')] * 2
+        pysolr.Solr().delete.assert_has_calls(calls)
+
+    @mock.patch('allura.lib.solr.pysolr')
+    def test_commit(self, pysolr):
+        servers = ['server1', 'server2']
+        solr = Solr(servers, commit=False, commitWithin='10000')
+        solr.commit('arg')
+        pysolr.Solr().commit.assert_has_calls([mock.call('arg')] * 2)
         pysolr.reset_mock()
-        s.delete('bar', somekw='value')
-        pysolr.Solr.delete.assert_called_once_with(s, 'bar', commit=False,
-                somekw='value')
+        solr.commit('arg', kw='kw')
+        calls = [mock.call('arg', kw='kw')] * 2
+        pysolr.Solr().commit.assert_has_calls(calls)
+
+    @mock.patch('allura.lib.solr.pysolr')
+    def test_search(self, pysolr):
+        servers = ['server1', 'server2']
+        solr = Solr(servers, commit=False, commitWithin='10000')
+        solr.search('foo')
+        solr.query_server.search.assert_called_once_with('foo')
+        pysolr.reset_mock()
+        solr.search('bar', kw='kw')
+        solr.query_server.search.assert_called_once_with('bar', kw='kw')
+
 
 class TestSolarize(unittest.TestCase):
 

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/7a332c6e/Allura/development.ini
----------------------------------------------------------------------
diff --git a/Allura/development.ini b/Allura/development.ini
index 761876b..b2ac7bf 100644
--- a/Allura/development.ini
+++ b/Allura/development.ini
@@ -177,7 +177,10 @@ amqp.enabled = false
 openid.realm = http://localhost:8080/
 
 # SOLR setup
+# comma-separated list of servers to push changes to
 solr.server = http://localhost:8983/solr/core0
+# server to read from. Uses solr.server[0] if not specified.
+# solr.query_server =
 # commit on every add/delete?
 solr.commit = false
 # commit add operations within N ms


Mime
View raw message