incubator-allura-commits mailing list archives

Site index · List index
Message view « Date » · « Thread »
Top « Date » · « Thread »
From tvansteenbu...@apache.org
Subject [3/4] git commit: [#6276] add --solr-hosts option to reindex cmd
Date Fri, 07 Jun 2013 23:42:48 GMT
[#6276] add --solr-hosts option to reindex cmd


Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/55b86ebe
Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/55b86ebe
Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/55b86ebe

Branch: refs/heads/master
Commit: 55b86ebe115e451b97ee1317ba4579b0ee3a3bcd
Parents: 577b65f
Author: Dave Brondsema <dbrondsema@slashdotmedia.com>
Authored: Tue Jun 4 16:44:17 2013 +0000
Committer: Tim Van Steenburgh <tvansteenburgh@gmail.com>
Committed: Fri Jun 7 23:42:12 2013 +0000

----------------------------------------------------------------------
 Allura/allura/command/show_models.py |   17 +++++++++++++----
 Allura/allura/lib/app_globals.py     |   14 +++++---------
 Allura/allura/lib/solr.py            |   19 +++++++++++++++++--
 Allura/allura/tasks/index_tasks.py   |   16 +++++++++++++---
 Allura/allura/tests/test_commands.py |   18 ++++++++++++++++++
 5 files changed, 66 insertions(+), 18 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/55b86ebe/Allura/allura/command/show_models.py
----------------------------------------------------------------------
diff --git a/Allura/allura/command/show_models.py b/Allura/allura/command/show_models.py
index c4e49d1..68a55ac 100644
--- a/Allura/allura/command/show_models.py
+++ b/Allura/allura/command/show_models.py
@@ -68,6 +68,8 @@ class ReindexCommand(base.Command):
                       help='Run each individual index operation as a background task.  '
                            'Note: this is often better, since tasks have "request" objects
'
                            'which are needed for some markdown macros to run properly')
+    parser.add_option('--solr-hosts', dest='solr_hosts',
+                      help='Override the solr host(s) to post to.  Comma-separated list of
solr server URLs')
 
     def command(self):
         from allura import model as M
@@ -88,6 +90,11 @@ class ReindexCommand(base.Command):
         if not self.options.solr and not self.options.refs:
             self.options.solr = self.options.refs = True
 
+        if self.options.solr_hosts:
+            self.add_artifact_kwargs = {'solr_hosts': self.options.solr_hosts.split(',')}
+        else:
+            self.add_artifact_kwargs = {}
+
         for projects in utils.chunked_find(M.Project, q_project):
             for p in projects:
                 c.project = p
@@ -123,8 +130,9 @@ class ReindexCommand(base.Command):
                             self._chunked_add_artifacts(ref_ids)
                         else:
                             add_artifacts(ref_ids,
-                                    update_solr=self.options.solr,
-                                    update_refs=self.options.refs)
+                                          update_solr=self.options.solr,
+                                          update_refs=self.options.refs,
+                                          **self.add_artifact_kwargs)
                     except CompoundError, err:
                         base.log.exception('Error indexing artifacts:\n%r', err)
                         base.log.error('%s', err.format_error())
@@ -146,8 +154,9 @@ class ReindexCommand(base.Command):
         """
         try:
             add_artifacts.post(chunk,
-                    update_solr=self.options.solr,
-                    update_refs=self.options.refs)
+                               update_solr=self.options.solr,
+                               update_refs=self.options.refs,
+                               **self.add_artifact_kwargs)
         except InvalidDocument as e:
             # there are many types of InvalidDocument, only recurse if its expected to help
             if str(e).startswith('BSON document too large'):

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/55b86ebe/Allura/allura/lib/app_globals.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py
index 26e0854..bdcacf7 100644
--- a/Allura/allura/lib/app_globals.py
+++ b/Allura/allura/lib/app_globals.py
@@ -57,7 +57,7 @@ from allura.lib import helpers as h
 from allura.lib.widgets import analytics
 from allura.lib.security import Credentials
 from allura.lib.async import Connection, MockAMQ
-from allura.lib.solr import Solr, MockSOLR
+from allura.lib.solr import MockSOLR, make_solr_from_config
 from allura.lib.zarkov_helpers import ZarkovClient, zmq
 
 log = logging.getLogger(__name__)
@@ -103,14 +103,10 @@ class Globals(object):
         if asbool(config.get('solr.mock')):
             self.solr = self.solr_short_timeout = MockSOLR()
         elif self.solr_server:
-            self.solr = Solr(self.solr_server, self.solr_query_server,
-                             commit=asbool(config.get('solr.commit', True)),
-                             commitWithin=config.get('solr.commitWithin'),
-                             timeout=int(config.get('solr.long_timeout', 60)))
-            self.solr_short_timeout = Solr(self.solr_server, self.solr_query_server,
-                                           commit=asbool(config.get('solr.commit', True)),
-                                           commitWithin=config.get('solr.commitWithin'),
-                                           timeout=int(config.get('solr.short_timeout', 10)))
+            self.solr = make_solr_from_config(self.solr_server, self.solr_query_server)
+            self.solr_short_timeout = make_solr_from_config(
+                self.solr_server, self.solr_query_server,
+                timeout=int(config.get('solr.short_timeout', 10)))
         else: # pragma no cover
             self.solr = None
             self.solr_short_timeout = None

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/55b86ebe/Allura/allura/lib/solr.py
----------------------------------------------------------------------
diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py
index e1e9567..4fa0c77 100644
--- a/Allura/allura/lib/solr.py
+++ b/Allura/allura/lib/solr.py
@@ -16,8 +16,24 @@
 #       under the License.
 
 import shlex
+
+from tg import config
+from paste.deploy.converters import asbool
 import pysolr
-from pysolr import SolrError
+
+
+def make_solr_from_config(push_servers, query_server=None, **kwargs):
+    """
+    Make a :class:`Solr <Solr>` instance from config defaults.  Use
+    `**kwargs` to override any value
+    """
+    solr_kwargs = dict(
+        commit=asbool(config.get('solr.commit', True)),
+        commitWithin=config.get('solr.commitWithin'),
+        timeout=int(config.get('solr.long_timeout', 60)),
+    )
+    solr_kwargs.update(kwargs)
+    return Solr(push_servers, query_server, **solr_kwargs)
 
 
 class Solr(object):
@@ -131,4 +147,3 @@ class MockSOLR(object):
         elif kwargs.get('q', None):
             for doc in self.search(kwargs['q']):
                 self.delete(id=doc['id'])
-

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/55b86ebe/Allura/allura/tasks/index_tasks.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tasks/index_tasks.py b/Allura/allura/tasks/index_tasks.py
index ac5049c..27ada83 100644
--- a/Allura/allura/tasks/index_tasks.py
+++ b/Allura/allura/tasks/index_tasks.py
@@ -23,14 +23,24 @@ from pylons import app_globals as g
 
 from allura.lib.decorators import task
 from allura.lib.exceptions import CompoundError
+from allura.lib.solr import make_solr_from_config
 
 log = logging.getLogger(__name__)
 
 @task
-def add_artifacts(ref_ids, update_solr=True, update_refs=True):
-    '''Add the referenced artifacts to SOLR and shortlinks'''
+def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None):
+    '''
+    Add the referenced artifacts to SOLR and shortlinks.
+
+    :param solr_hosts: a list of solr hosts to use instead of the defaults
+    :type solr_hosts: [str]
+    '''
     from allura import model as M
     from allura.lib.search import find_shortlinks, solarize
+    if solr_hosts:
+        solr = make_solr_from_config(solr_hosts)
+    else:
+        solr = g.solr
     exceptions = []
     solr_updates = []
     with _indexing_disabled(M.session.artifact_orm_session._get()):
@@ -50,7 +60,7 @@ def add_artifacts(ref_ids, update_solr=True, update_refs=True):
             except Exception:
                 log.error('Error indexing artifact %s', ref._id)
                 exceptions.append(sys.exc_info())
-        g.solr.add(solr_updates)
+        solr.add(solr_updates)
 
     if len(exceptions) == 1:
         raise exceptions[0][0], exceptions[0][1], exceptions[0][2]

http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/55b86ebe/Allura/allura/tests/test_commands.py
----------------------------------------------------------------------
diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py
index fee6e21..1d72ff6 100644
--- a/Allura/allura/tests/test_commands.py
+++ b/Allura/allura/tests/test_commands.py
@@ -362,6 +362,21 @@ class TestReindexCommand(object):
         cmd.run([test_config, '-p', 'test', '--solr', '--skip-solr-delete'])
         assert not g.solr.delete.called, 'solr.delete() must not be called'
 
+    @patch('pysolr.Solr')
+    def test_solr_hosts_1(self, Solr):
+        cmd = show_models.ReindexCommand('reindex')
+        cmd.run([test_config, '-p', 'test', '--solr', '--solr-hosts=http://blah.com/solr/forge'])
+        assert_equal(Solr.call_args[0][0], 'http://blah.com/solr/forge')
+
+    @patch('pysolr.Solr')
+    def test_solr_hosts_list(self, Solr):
+        cmd = show_models.ReindexCommand('reindex')
+        cmd.run([test_config, '-p', 'test', '--solr', '--solr-hosts=http://blah.com/solr/forge,https://other.net/solr/forge'])
+        # check constructors of first and second Solr() instantiations
+        assert_equal(set([Solr.call_args_list[0][0][0], Solr.call_args_list[1][0][0]]),
+                     set(['http://blah.com/solr/forge', 'https://other.net/solr/forge'])
+                     )
+
     @patch('allura.command.show_models.utils')
     def test_project_regex(self, utils):
         cmd = show_models.ReindexCommand('reindex')
@@ -373,6 +388,7 @@ class TestReindexCommand(object):
     def test_chunked_add_artifacts(self, add_artifacts):
         cmd = show_models.ReindexCommand('reindex')
         cmd.options = Mock()
+        cmd.add_artifact_kwargs = {}
         ref_ids = list(range(100 * 1000 * 2 + 20))
         cmd._chunked_add_artifacts(ref_ids)
         assert_equal(len(add_artifacts.post.call_args_list), 3)
@@ -389,6 +405,7 @@ class TestReindexCommand(object):
         add_artifacts.post.side_effect = on_post
         cmd = show_models.ReindexCommand('reindex')
         cmd.options = Mock()
+        cmd.add_artifact_kwargs = {}
         cmd._post_add_artifacts(range(5))
         kw = {'update_solr': cmd.options.solr, 'update_refs': cmd.options.refs}
         expected = [
@@ -411,5 +428,6 @@ class TestReindexCommand(object):
         add_artifacts.post.side_effect = on_post
         cmd = show_models.ReindexCommand('reindex')
         cmd.options = Mock()
+        cmd.add_artifact_kwargs = {}
         with td.raises(pymongo.errors.InvalidDocument):
             cmd._post_add_artifacts(range(5))


Mime
View raw message