Return-Path: X-Original-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-allura-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id 395A310BCC for ; Fri, 7 Jun 2013 23:42:47 +0000 (UTC) Received: (qmail 45345 invoked by uid 500); 7 Jun 2013 23:42:47 -0000 Delivered-To: apmail-incubator-allura-commits-archive@incubator.apache.org Received: (qmail 45326 invoked by uid 500); 7 Jun 2013 23:42:47 -0000 Mailing-List: contact allura-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: allura-dev@incubator.apache.org Delivered-To: mailing list allura-commits@incubator.apache.org Received: (qmail 45295 invoked by uid 99); 7 Jun 2013 23:42:47 -0000 Received: from tyr.zones.apache.org (HELO tyr.zones.apache.org) (140.211.11.114) by apache.org (qpsmtpd/0.29) with ESMTP; Fri, 07 Jun 2013 23:42:47 +0000 Received: by tyr.zones.apache.org (Postfix, from userid 65534) id E09FB8A0999; Fri, 7 Jun 2013 23:42:46 +0000 (UTC) Content-Type: text/plain; charset="us-ascii" MIME-Version: 1.0 Content-Transfer-Encoding: 7bit From: tvansteenburgh@apache.org To: allura-commits@incubator.apache.org Date: Fri, 07 Jun 2013 23:42:48 -0000 Message-Id: In-Reply-To: <61baac7c86a147428af0f529ad4344ad@git.apache.org> References: <61baac7c86a147428af0f529ad4344ad@git.apache.org> X-Mailer: ASF-Git Admin Mailer Subject: [3/4] git commit: [#6276] add --solr-hosts option to reindex cmd [#6276] add --solr-hosts option to reindex cmd Project: http://git-wip-us.apache.org/repos/asf/incubator-allura/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-allura/commit/55b86ebe Tree: http://git-wip-us.apache.org/repos/asf/incubator-allura/tree/55b86ebe Diff: http://git-wip-us.apache.org/repos/asf/incubator-allura/diff/55b86ebe Branch: refs/heads/master Commit: 55b86ebe115e451b97ee1317ba4579b0ee3a3bcd Parents: 577b65f Author: Dave Brondsema Authored: Tue Jun 4 16:44:17 2013 +0000 Committer: Tim Van Steenburgh Committed: Fri Jun 7 23:42:12 2013 +0000 ---------------------------------------------------------------------- Allura/allura/command/show_models.py | 17 +++++++++++++---- Allura/allura/lib/app_globals.py | 14 +++++--------- Allura/allura/lib/solr.py | 19 +++++++++++++++++-- Allura/allura/tasks/index_tasks.py | 16 +++++++++++++--- Allura/allura/tests/test_commands.py | 18 ++++++++++++++++++ 5 files changed, 66 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/55b86ebe/Allura/allura/command/show_models.py ---------------------------------------------------------------------- diff --git a/Allura/allura/command/show_models.py b/Allura/allura/command/show_models.py index c4e49d1..68a55ac 100644 --- a/Allura/allura/command/show_models.py +++ b/Allura/allura/command/show_models.py @@ -68,6 +68,8 @@ class ReindexCommand(base.Command): help='Run each individual index operation as a background task. ' 'Note: this is often better, since tasks have "request" objects ' 'which are needed for some markdown macros to run properly') + parser.add_option('--solr-hosts', dest='solr_hosts', + help='Override the solr host(s) to post to. Comma-separated list of solr server URLs') def command(self): from allura import model as M @@ -88,6 +90,11 @@ class ReindexCommand(base.Command): if not self.options.solr and not self.options.refs: self.options.solr = self.options.refs = True + if self.options.solr_hosts: + self.add_artifact_kwargs = {'solr_hosts': self.options.solr_hosts.split(',')} + else: + self.add_artifact_kwargs = {} + for projects in utils.chunked_find(M.Project, q_project): for p in projects: c.project = p @@ -123,8 +130,9 @@ class ReindexCommand(base.Command): self._chunked_add_artifacts(ref_ids) else: add_artifacts(ref_ids, - update_solr=self.options.solr, - update_refs=self.options.refs) + update_solr=self.options.solr, + update_refs=self.options.refs, + **self.add_artifact_kwargs) except CompoundError, err: base.log.exception('Error indexing artifacts:\n%r', err) base.log.error('%s', err.format_error()) @@ -146,8 +154,9 @@ class ReindexCommand(base.Command): """ try: add_artifacts.post(chunk, - update_solr=self.options.solr, - update_refs=self.options.refs) + update_solr=self.options.solr, + update_refs=self.options.refs, + **self.add_artifact_kwargs) except InvalidDocument as e: # there are many types of InvalidDocument, only recurse if its expected to help if str(e).startswith('BSON document too large'): http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/55b86ebe/Allura/allura/lib/app_globals.py ---------------------------------------------------------------------- diff --git a/Allura/allura/lib/app_globals.py b/Allura/allura/lib/app_globals.py index 26e0854..bdcacf7 100644 --- a/Allura/allura/lib/app_globals.py +++ b/Allura/allura/lib/app_globals.py @@ -57,7 +57,7 @@ from allura.lib import helpers as h from allura.lib.widgets import analytics from allura.lib.security import Credentials from allura.lib.async import Connection, MockAMQ -from allura.lib.solr import Solr, MockSOLR +from allura.lib.solr import MockSOLR, make_solr_from_config from allura.lib.zarkov_helpers import ZarkovClient, zmq log = logging.getLogger(__name__) @@ -103,14 +103,10 @@ class Globals(object): if asbool(config.get('solr.mock')): self.solr = self.solr_short_timeout = MockSOLR() elif self.solr_server: - self.solr = Solr(self.solr_server, self.solr_query_server, - commit=asbool(config.get('solr.commit', True)), - commitWithin=config.get('solr.commitWithin'), - timeout=int(config.get('solr.long_timeout', 60))) - self.solr_short_timeout = Solr(self.solr_server, self.solr_query_server, - commit=asbool(config.get('solr.commit', True)), - commitWithin=config.get('solr.commitWithin'), - timeout=int(config.get('solr.short_timeout', 10))) + self.solr = make_solr_from_config(self.solr_server, self.solr_query_server) + self.solr_short_timeout = make_solr_from_config( + self.solr_server, self.solr_query_server, + timeout=int(config.get('solr.short_timeout', 10))) else: # pragma no cover self.solr = None self.solr_short_timeout = None http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/55b86ebe/Allura/allura/lib/solr.py ---------------------------------------------------------------------- diff --git a/Allura/allura/lib/solr.py b/Allura/allura/lib/solr.py index e1e9567..4fa0c77 100644 --- a/Allura/allura/lib/solr.py +++ b/Allura/allura/lib/solr.py @@ -16,8 +16,24 @@ # under the License. import shlex + +from tg import config +from paste.deploy.converters import asbool import pysolr -from pysolr import SolrError + + +def make_solr_from_config(push_servers, query_server=None, **kwargs): + """ + Make a :class:`Solr ` instance from config defaults. Use + `**kwargs` to override any value + """ + solr_kwargs = dict( + commit=asbool(config.get('solr.commit', True)), + commitWithin=config.get('solr.commitWithin'), + timeout=int(config.get('solr.long_timeout', 60)), + ) + solr_kwargs.update(kwargs) + return Solr(push_servers, query_server, **solr_kwargs) class Solr(object): @@ -131,4 +147,3 @@ class MockSOLR(object): elif kwargs.get('q', None): for doc in self.search(kwargs['q']): self.delete(id=doc['id']) - http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/55b86ebe/Allura/allura/tasks/index_tasks.py ---------------------------------------------------------------------- diff --git a/Allura/allura/tasks/index_tasks.py b/Allura/allura/tasks/index_tasks.py index ac5049c..27ada83 100644 --- a/Allura/allura/tasks/index_tasks.py +++ b/Allura/allura/tasks/index_tasks.py @@ -23,14 +23,24 @@ from pylons import app_globals as g from allura.lib.decorators import task from allura.lib.exceptions import CompoundError +from allura.lib.solr import make_solr_from_config log = logging.getLogger(__name__) @task -def add_artifacts(ref_ids, update_solr=True, update_refs=True): - '''Add the referenced artifacts to SOLR and shortlinks''' +def add_artifacts(ref_ids, update_solr=True, update_refs=True, solr_hosts=None): + ''' + Add the referenced artifacts to SOLR and shortlinks. + + :param solr_hosts: a list of solr hosts to use instead of the defaults + :type solr_hosts: [str] + ''' from allura import model as M from allura.lib.search import find_shortlinks, solarize + if solr_hosts: + solr = make_solr_from_config(solr_hosts) + else: + solr = g.solr exceptions = [] solr_updates = [] with _indexing_disabled(M.session.artifact_orm_session._get()): @@ -50,7 +60,7 @@ def add_artifacts(ref_ids, update_solr=True, update_refs=True): except Exception: log.error('Error indexing artifact %s', ref._id) exceptions.append(sys.exc_info()) - g.solr.add(solr_updates) + solr.add(solr_updates) if len(exceptions) == 1: raise exceptions[0][0], exceptions[0][1], exceptions[0][2] http://git-wip-us.apache.org/repos/asf/incubator-allura/blob/55b86ebe/Allura/allura/tests/test_commands.py ---------------------------------------------------------------------- diff --git a/Allura/allura/tests/test_commands.py b/Allura/allura/tests/test_commands.py index fee6e21..1d72ff6 100644 --- a/Allura/allura/tests/test_commands.py +++ b/Allura/allura/tests/test_commands.py @@ -362,6 +362,21 @@ class TestReindexCommand(object): cmd.run([test_config, '-p', 'test', '--solr', '--skip-solr-delete']) assert not g.solr.delete.called, 'solr.delete() must not be called' + @patch('pysolr.Solr') + def test_solr_hosts_1(self, Solr): + cmd = show_models.ReindexCommand('reindex') + cmd.run([test_config, '-p', 'test', '--solr', '--solr-hosts=http://blah.com/solr/forge']) + assert_equal(Solr.call_args[0][0], 'http://blah.com/solr/forge') + + @patch('pysolr.Solr') + def test_solr_hosts_list(self, Solr): + cmd = show_models.ReindexCommand('reindex') + cmd.run([test_config, '-p', 'test', '--solr', '--solr-hosts=http://blah.com/solr/forge,https://other.net/solr/forge']) + # check constructors of first and second Solr() instantiations + assert_equal(set([Solr.call_args_list[0][0][0], Solr.call_args_list[1][0][0]]), + set(['http://blah.com/solr/forge', 'https://other.net/solr/forge']) + ) + @patch('allura.command.show_models.utils') def test_project_regex(self, utils): cmd = show_models.ReindexCommand('reindex') @@ -373,6 +388,7 @@ class TestReindexCommand(object): def test_chunked_add_artifacts(self, add_artifacts): cmd = show_models.ReindexCommand('reindex') cmd.options = Mock() + cmd.add_artifact_kwargs = {} ref_ids = list(range(100 * 1000 * 2 + 20)) cmd._chunked_add_artifacts(ref_ids) assert_equal(len(add_artifacts.post.call_args_list), 3) @@ -389,6 +405,7 @@ class TestReindexCommand(object): add_artifacts.post.side_effect = on_post cmd = show_models.ReindexCommand('reindex') cmd.options = Mock() + cmd.add_artifact_kwargs = {} cmd._post_add_artifacts(range(5)) kw = {'update_solr': cmd.options.solr, 'update_refs': cmd.options.refs} expected = [ @@ -411,5 +428,6 @@ class TestReindexCommand(object): add_artifacts.post.side_effect = on_post cmd = show_models.ReindexCommand('reindex') cmd.options = Mock() + cmd.add_artifact_kwargs = {} with td.raises(pymongo.errors.InvalidDocument): cmd._post_add_artifacts(range(5))