Return-Path: X-Original-To: apmail-incubator-bloodhound-commits-archive@minotaur.apache.org Delivered-To: apmail-incubator-bloodhound-commits-archive@minotaur.apache.org Received: from mail.apache.org (hermes.apache.org [140.211.11.3]) by minotaur.apache.org (Postfix) with SMTP id D5E7CD5E1 for ; Thu, 10 Jan 2013 13:33:21 +0000 (UTC) Received: (qmail 73622 invoked by uid 500); 10 Jan 2013 13:33:21 -0000 Delivered-To: apmail-incubator-bloodhound-commits-archive@incubator.apache.org Received: (qmail 73585 invoked by uid 500); 10 Jan 2013 13:33:21 -0000 Mailing-List: contact bloodhound-commits-help@incubator.apache.org; run by ezmlm Precedence: bulk List-Help: List-Unsubscribe: List-Post: List-Id: Reply-To: bloodhound-dev@incubator.apache.org Delivered-To: mailing list bloodhound-commits@incubator.apache.org Received: (qmail 73574 invoked by uid 99); 10 Jan 2013 13:33:21 -0000 Received: from athena.apache.org (HELO athena.apache.org) (140.211.11.136) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 Jan 2013 13:33:21 +0000 X-ASF-Spam-Status: No, hits=-2000.0 required=5.0 tests=ALL_TRUSTED X-Spam-Check-By: apache.org Received: from [140.211.11.4] (HELO eris.apache.org) (140.211.11.4) by apache.org (qpsmtpd/0.29) with ESMTP; Thu, 10 Jan 2013 13:33:17 +0000 Received: from eris.apache.org (localhost [127.0.0.1]) by eris.apache.org (Postfix) with ESMTP id 3018D238896F; Thu, 10 Jan 2013 13:32:58 +0000 (UTC) Content-Type: text/plain; charset="utf-8" MIME-Version: 1.0 Content-Transfer-Encoding: 8bit Subject: svn commit: r1431350 [1/2] - in /incubator/bloodhound/trunk/bloodhound_search: ./ bhsearch/ bhsearch/templates/ bhsearch/tests/ Date: Thu, 10 Jan 2013 13:32:57 -0000 To: bloodhound-commits@incubator.apache.org From: peter@apache.org X-Mailer: svnmailer-1.0.8-patched Message-Id: <20130110133258.3018D238896F@eris.apache.org> X-Virus-Checked: Checked by ClamAV on apache.org Author: peter Date: Thu Jan 10 13:32:56 2013 New Revision: 1431350 URL: http://svn.apache.org/viewvc?rev=1431350&view=rev Log: #BEP-0004: prototype phase 2 patch from Andrej Added: incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/wiki_search.py incubator/bloodhound/trunk/bloodhound_search/bhsearch/wiki_search.py Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/api.py incubator/bloodhound/trunk/bloodhound_search/bhsearch/query_parser.py incubator/bloodhound/trunk/bloodhound_search/bhsearch/templates/bhsearch.html incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/__init__.py incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/api.py incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/index_with_whoosh.py incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/ticket_search.py incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/utils.py incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/web_ui.py incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/whoosh_backend.py incubator/bloodhound/trunk/bloodhound_search/bhsearch/ticket_search.py incubator/bloodhound/trunk/bloodhound_search/bhsearch/web_ui.py incubator/bloodhound/trunk/bloodhound_search/bhsearch/whoosh_backend.py incubator/bloodhound/trunk/bloodhound_search/setup.py Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/api.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/api.py?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/api.py (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/api.py Thu Jan 10 13:32:56 2013 @@ -27,6 +27,13 @@ ASC = "asc" DESC = "desc" SCORE = "score" +class IndexFields(object): + TYPE = "type" + ID = "id" + TIME = 'time' + AUTHOR = 'author' + CONTENT = 'content' + class QueryResult(object): def __init__(self): self.hits = 0 @@ -35,48 +42,50 @@ class QueryResult(object): self.offset = 0 self.docs = [] self.facets = None + self.debug = {} class ISearchBackend(Interface): """Extension point interface for search backend systems. """ - def add_doc(self, doc, commit=True): +# def add_doc(self, doc, **kwargs): + def add_doc(doc, **kwargs): """ Called when new document instance must be added - - :param doc: document to add - :param commit: flag if commit should be automatically called """ - def delete_doc(self, doc, commit=True): + def delete_doc(type, id, **kwargs): """ Delete document from index + """ - :param doc: document to delete - :param commit: flag if commit should be automatically called + def optimize(): + """ + Optimize index if needed """ - def commit(self): + def commit(optimize, **kwargs): """ - Commits changes + Commit changes """ - def optimize(self): + def cancel(**kwargs): """ - Optimize index if needed + Cancel changes if possible """ - def recreate_index(self): + def recreate_index(): """ Create a new index, if index exists, it will be deleted """ - def open_or_create_index_if_missing(self): + def open_or_create_index_if_missing(): """ Open existing index, if index does not exist, create new one """ - def query(self, query, sort = None, fields = None, boost = None, filters = None, + + def query(query, sort = None, fields = None, boost = None, filter = None, facets = None, pagenum = 1, pagelen = 20): """ Perform query implementation @@ -85,38 +94,42 @@ class ISearchBackend(Interface): :param sort: :param fields: :param boost: - :param filters: + :param filter: :param facets: :param pagenum: :param pagelen: - :return: TBD!!! + :return: ResultsPage """ - pass + + def start_operation(self): + """Used to get arguments for batch operation withing single commit""" + +class IIndexParticipant(Interface): + """Extension point interface for components that should be searched. + """ + + def get_entries_for_index(): + """List entities for index creation""" class ISearchParticipant(Interface): """Extension point interface for components that should be searched. """ + def format_search_results(contents): + """Called to see if the module wants to format the search results.""" def get_search_filters(req): """Called when we want to build the list of components with search. Passes the request object to do permission checking.""" - pass - def build_search_index(backend): - """Called when we want to rebuild the entire index. - :type backend: ISearchBackend - """ - pass - - def format_search_results(contents): - """Called to see if the module wants to format the search results.""" + def get_title(): + """Return resource title""" class IQueryParser(Interface): """Extension point for Bloodhound Search query parser. """ def parse(query_string, req = None): - pass + """Parse query from string""" class BloodhoundSearchApi(Component): """Implements core indexing functionality, provides methods for @@ -132,10 +145,11 @@ class BloodhoundSearchApi(Component): 'Name of the component implementing Bloodhound Search query \ parser.') - search_participants = ExtensionPoint(ISearchParticipant) + index_participants = ExtensionPoint(IIndexParticipant) - def query(self, query, req = None, sort = None, fields = None, boost = None, filters = None, - facets = None, pagenum = 1, pagelen = 20): + def query(self, query, sort = None, fields = None, + boost = None, filter = None, + facets = None, pagenum = 1, pagelen = 20): """Return query result from an underlying search backend. Arguments: @@ -144,9 +158,9 @@ class BloodhoundSearchApi(Component): :param sort: optional sorting :param boost: optional list of fields with boost values e.g. {“id”: 1000, “subject” :100, “description”:10}. - :param filters: optional list of terms. Usually can be cached by underlying - search framework. For example {“type”: “wiki”} - :param facets: optional list of facet terms, can be field or expression. + :param filter: optional list of terms. Usually can be cached by + underlying search framework. For example {“type”: “wiki”} + :param facets: optional list of facet terms, can be field or expression :param page: paging support :param pagelen: paging support @@ -154,18 +168,17 @@ class BloodhoundSearchApi(Component): """ self.env.log.debug("Receive query request: %s", locals()) + parsed_query = self.parser.parse(query) + # TODO: add query parsers and meta keywords post-parsing # TODO: apply security filters - parsed_query = self.parser.parse(query, req) - - #some backend-independent logic will come here... query_result = self.backend.query( query = parsed_query, sort = sort, fields = fields, - filters = filters, + filter = filter, facets = facets, pagenum = pagenum, pagelen = pagelen, @@ -175,39 +188,64 @@ class BloodhoundSearchApi(Component): def rebuild_index(self): - """Delete the index if it exists. Then create a new full index.""" + """Rebuild underlying index""" self.log.info('Rebuilding the search index.') self.backend.recreate_index() + operation_data = self.backend.start_operation() + try: + for participant in self.index_participants: + docs = participant.get_entries_for_index() + for doc in docs: + self.backend.add_doc(doc, **operation_data) + self.backend.commit(True, **operation_data) + except: + self.backend.cancel(**operation_data) + raise + + def change_doc_id(self, doc, old_id): + operation_data = self.backend.start_operation() + try: + self.backend.delete_doc( + doc[IndexFields.TYPE], + old_id, + **operation_data) + self.backend.add_doc(doc, **operation_data) + self.backend.commit(False, **operation_data) + except: + self.backend.cancel(**operation_data) + raise - for participant in self.search_participants: - participant.build_search_index(self.backend) - self.backend.commit() - self.backend.optimize() - - #Erase the index if it exists. Then create a new index from scratch. - - #erase ticket - #call reindex for each resource - #commit - pass def optimize(self): """Optimize underlying index""" - pass + self.backend.optimize() def add_doc(self, doc): """Add a document to underlying search backend. The doc must be dictionary with obligatory "type" field """ - self.backend.add_doc(doc) + operation_data = self.backend.start_operation() + try: + self.backend.add_doc(doc, **operation_data) + self.backend.commit(False, **operation_data) + except: + self.backend.cancel(**operation_data) + raise + def delete_doc(self, type, id): """Add a document from underlying search backend. The doc must be dictionary with obligatory "type" field """ - pass + operation_data = self.backend.start_operation() + try: + self.backend.delete_doc(type, id, **operation_data) + self.backend.commit(False, **operation_data) + except: + self.backend.cancel(**operation_data) + raise Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/query_parser.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/query_parser.py?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/query_parser.py (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/query_parser.py Thu Jan 10 13:32:56 2013 @@ -23,12 +23,12 @@ r"""Provides Bloodhound Search query par from bhsearch.api import IQueryParser from bhsearch.whoosh_backend import WhooshBackend from trac.core import Component, implements -from whoosh.qparser import QueryParser, DisMaxParser, MultifieldParser +from whoosh.qparser import MultifieldParser class DefaultQueryParser(Component): implements(IQueryParser) - def parse(self, query_string, req = None): + def parse(self, query_string): #todo: make field boost configurable e.g. read from config setting #this is prototype implementation ,the fields boost must be tuned later field_boosts = dict( Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/templates/bhsearch.html URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/templates/bhsearch.html?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/templates/bhsearch.html (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/templates/bhsearch.html Thu Jan 10 13:32:56 2013 @@ -43,16 +43,25 @@
-

This page provides prototype functionality. Implementation is coming...

+

This page provides prototype functionality.

+

+
+ +
+

Results (${results.displayed_items()}) @@ -60,18 +69,7 @@
- -
- -
- +
${result.title}
${result.excerpt}
Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/__init__.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/__init__.py?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/__init__.py (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/__init__.py Thu Jan 10 13:32:56 2013 @@ -17,17 +17,17 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -import doctest import unittest -from bhsearch.tests import whoosh_backend, index_with_whoosh, web_ui, ticket_search, api +from bhsearch.tests import whoosh_backend, index_with_whoosh, web_ui, ticket_search, api, wiki_search def suite(): suite = unittest.TestSuite() suite.addTest(index_with_whoosh.suite()) suite.addTest(whoosh_backend.suite()) suite.addTest(web_ui.suite()) - suite.addTest(ticket_search.suite()) suite.addTest(api.suite()) + suite.addTest(ticket_search.suite()) + suite.addTest(wiki_search.suite()) return suite if __name__ == '__main__': Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/api.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/api.py?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/api.py (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/api.py Thu Jan 10 13:32:56 2013 @@ -17,9 +17,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from datetime import datetime, timedelta -from pprint import pprint - import unittest import tempfile import shutil @@ -29,7 +26,7 @@ from bhsearch.tests.utils import BaseBlo from bhsearch.ticket_search import TicketSearchParticipant from bhsearch.whoosh_backend import WhooshBackend -from trac.test import EnvironmentStub, Mock, MockPerm +from trac.test import EnvironmentStub from trac.ticket.api import TicketSystem @@ -37,12 +34,12 @@ class ApiQueryWithWhooshTestCase(BaseBlo def setUp(self): self.env = EnvironmentStub(enable=['bhsearch.*']) self.env.path = tempfile.mkdtemp('bhsearch-tempenv') + self.env.config.set('bhsearch', 'silence_on_error', "False") self.ticket_system = TicketSystem(self.env) self.whoosh_backend = WhooshBackend(self.env) self.whoosh_backend.recreate_index() self.search_api = BloodhoundSearchApi(self.env) self.ticket_participant = TicketSearchParticipant(self.env) - self.ticket_system = TicketSystem(self.env) self.query_parser = DefaultQueryParser(self.env) def tearDown(self): @@ -103,17 +100,19 @@ class ApiQueryWithWhooshTestCase(BaseBlo docs = results.docs self.assertEqual("summary1 keyword", docs[0]["summary"]) - def test_can_search_id_and_summary(self): - #arrange - self.insert_ticket("summary1") - self.insert_ticket("summary2 1") - #act - results = self.search_api.query("1") - self.print_result(results) - #assert - self.assertEqual(2, results.hits) - docs = results.docs - self.assertEqual("summary1", docs[0]["summary"]) +#TODO: check this later +# @unittest.skip("Check with Whoosh community") +# def test_can_search_id_and_summary(self): +# #arrange +# self.insert_ticket("summary1") +# self.insert_ticket("summary2 1") +# #act +# results = self.search_api.query("1") +# self.print_result(results) +# #assert +# self.assertEqual(2, results.hits) +# docs = results.docs +# self.assertEqual("summary1", docs[0]["summary"]) def suite(): return unittest.makeSuite(ApiQueryWithWhooshTestCase, 'test') Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/index_with_whoosh.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/index_with_whoosh.py?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/index_with_whoosh.py (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/index_with_whoosh.py Thu Jan 10 13:32:56 2013 @@ -24,9 +24,10 @@ import tempfile import shutil from bhsearch.api import BloodhoundSearchApi from bhsearch.tests.utils import BaseBloodhoundSearchTest -from bhsearch.ticket_search import TicketSearchParticipant +from bhsearch.ticket_search import TicketIndexer from bhsearch.whoosh_backend import WhooshBackend +from bhsearch.wiki_search import WikiIndexer from trac.test import EnvironmentStub from trac.ticket.api import TicketSystem @@ -35,11 +36,11 @@ class IndexWhooshTestCase(BaseBloodhound def setUp(self): self.env = EnvironmentStub(enable=['bhsearch.*']) self.env.path = tempfile.mkdtemp('bhsearch-tempenv') -# self.perm = PermissionSystem(self.env) self.whoosh_backend = WhooshBackend(self.env) self.whoosh_backend.recreate_index() self.search_api = BloodhoundSearchApi(self.env) - self.ticket_participant = TicketSearchParticipant(self.env) + self.ticket_indexer = TicketIndexer(self.env) + self.wiki_indexer = WikiIndexer(self.env) self.ticket_system = TicketSystem(self.env) def tearDown(self): @@ -49,7 +50,7 @@ class IndexWhooshTestCase(BaseBloodhound def test_can_index_ticket(self): ticket = self.create_dummy_ticket() ticket.id = "1" - self.ticket_participant.ticket_created(ticket) + self.ticket_indexer.ticket_created(ticket) results = self.search_api.query("*:*") self.print_result(results) @@ -64,7 +65,18 @@ class IndexWhooshTestCase(BaseBloodhound self.print_result(results) self.assertEqual(1, results.hits) - def test_can_reindex(self): + def test_can_reindex_twice(self): + self.insert_ticket("t1") + self.whoosh_backend.recreate_index() + #act + self.search_api.rebuild_index() + #just to test that index was re-created + self.search_api.rebuild_index() + #assert + results = self.search_api.query("*:*") + self.assertEqual(1, results.hits) + + def test_can_reindex_tickets(self): self.insert_ticket("t1") self.insert_ticket("t2") self.insert_ticket("t3") @@ -76,6 +88,28 @@ class IndexWhooshTestCase(BaseBloodhound self.print_result(results) self.assertEqual(3, results.hits) + def test_can_reindex_wiki(self): + self.insert_wiki("page1", "some text") + self.insert_wiki("page2", "some text") + self.whoosh_backend.recreate_index() + #act + self.search_api.rebuild_index() + #assert + results = self.search_api.query("*:*") + self.print_result(results) + self.assertEqual(2, results.hits) + + def test_can_reindex_mixed_types(self): + self.insert_wiki("page1", "some text") + self.insert_ticket("t1") + self.whoosh_backend.recreate_index() + #act + self.search_api.rebuild_index() + #assert + results = self.search_api.query("*:*") + self.print_result(results) + self.assertEqual(2, results.hits) + def suite(): return unittest.makeSuite(IndexWhooshTestCase, 'test') Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/ticket_search.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/ticket_search.py?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/ticket_search.py (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/ticket_search.py Thu Jan 10 13:32:56 2013 @@ -21,40 +21,36 @@ from datetime import datetime import unittest import tempfile -import shutil -from bhsearch.api import BloodhoundSearchApi from bhsearch.tests.utils import BaseBloodhoundSearchTest -from bhsearch.ticket_search import TicketSearchParticipant +from bhsearch.ticket_search import TicketIndexer -from bhsearch.whoosh_backend import WhooshBackend from trac.test import EnvironmentStub -from trac.ticket.api import TicketSystem -class TicketSearchSilenceOnExceptionTestCase(BaseBloodhoundSearchTest): +class TicketIndexerSilenceOnExceptionTestCase(BaseBloodhoundSearchTest): def setUp(self): self.env = EnvironmentStub( enable=['bhsearch.*'], path=tempfile.mkdtemp('bhsearch-tempenv'), ) - self.ticket_participant = TicketSearchParticipant(self.env) + self.ticket_indexer = TicketIndexer(self.env) def tearDown(self): pass def test_does_not_raise_exception_by_default(self): - self.ticket_participant.ticket_created(None) + self.ticket_indexer.ticket_created(None) def test_raise_exception_if_configured(self): self.env.config.set('bhsearch', 'silence_on_error', "False") self.assertRaises( Exception, - self.ticket_participant.ticket_created, + self.ticket_indexer.ticket_created, None) def suite(): - return unittest.makeSuite(TicketSearchSilenceOnExceptionTestCase, 'test') + return unittest.makeSuite(TicketIndexerSilenceOnExceptionTestCase, 'test') if __name__ == '__main__': unittest.main() Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/utils.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/utils.py?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/utils.py (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/utils.py Thu Jan 10 13:32:56 2013 @@ -24,6 +24,7 @@ Test utils methods import pprint import unittest from trac.ticket import Ticket +from trac.wiki import WikiPage class BaseBloodhoundSearchTest(unittest.TestCase): def print_result(self, result): @@ -48,3 +49,16 @@ class BaseBloodhoundSearchTest(unittest. ticket = self.create_ticket(summary, **kw) return ticket.insert() + def create_wiki(self, name, text, **kw): + page = WikiPage(self.env, name) + page.text = text + for k, v in kw.items(): + page[k] = v + return page + + def insert_wiki(self, name, text = None, **kw): + """Helper for inserting a ticket into the database""" + text = text or "Dummy text" + page = self.create_wiki(name, text, **kw) + return page.save("dummy author", "dummy comment", "::1") + Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/web_ui.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/web_ui.py?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/web_ui.py (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/web_ui.py Thu Jan 10 13:32:56 2013 @@ -17,7 +17,6 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from datetime import datetime, timedelta from pprint import pprint import unittest @@ -26,7 +25,7 @@ import shutil from bhsearch.api import BloodhoundSearchApi from bhsearch.tests.utils import BaseBloodhoundSearchTest from bhsearch.ticket_search import TicketSearchParticipant -from bhsearch.web_ui import BloodhoundSearchModule, SEARCH_PERMISSION +from bhsearch.web_ui import BloodhoundSearchModule, RequestParameters from bhsearch.whoosh_backend import WhooshBackend from trac.test import EnvironmentStub, Mock, MockPerm @@ -36,7 +35,7 @@ from trac.util.datefmt import FixedOffse from trac.util import format_datetime from trac.web import Href -BHSEARCH_URL = "/bhsearch" +BHSEARCH_URL = "/main/bhsearch" DEFAULT_DOCS_PER_PAGE = 10 class WebUiTestCaseWithWhoosh(BaseBloodhoundSearchTest): @@ -56,19 +55,9 @@ class WebUiTestCaseWithWhoosh(BaseBloodh self.req = Mock( perm=MockPerm(), chrome={'logo': {}}, - href=Href("/bhsearch"), + href=Href("/main"), args={}, ) -# self.req = Mock(href=self.env.href, authname='anonymous', tz=utc) -# self.req = Mock(base_path='/trac.cgi', path_info='', -# href=Href('/trac.cgi'), chrome={'logo': {}}, -# abs_href=Href('http://example.org/trac.cgi'), -# environ={}, perm=[], authname='-', args={}, tz=None, -# locale='', session=None, form_token=None) - -# self.req = Mock(href=self.env.href, abs_href=self.env.abs_href, tz=utc, -# perm=MockPerm()) -# def tearDown(self): shutil.rmtree(self.env.path) @@ -89,14 +78,14 @@ class WebUiTestCaseWithWhoosh(BaseBloodh self.assertEqual("", data["query"]) def test_can_process_query_empty_data(self): - self.req.args["q"] = "bla" + self.req.args[RequestParameters.QUERY] = "bla" data = self._process_request() self.assertEqual("bla", data["query"]) self.assertEqual([], data["results"].items) def test_can_process_first_page(self): - self._insert_docs(5) - self.req.args["q"] = "summary:test" + self._insert_tickets(5) + self.req.args[RequestParameters.QUERY] = "summary:test" data = self._process_request() self.assertEqual("summary:test", data["query"]) self.assertEqual(5, len(data["results"].items)) @@ -107,7 +96,7 @@ class WebUiTestCaseWithWhoosh(BaseBloodh ticket = Ticket(self.env, ticket_id) ticket_time = ticket.time_changed #act - self.req.args["q"] = "*:*" + self.req.args[RequestParameters.QUERY] = "*:*" data = self._process_request() result_items = data["results"].items #assert @@ -123,7 +112,7 @@ class WebUiTestCaseWithWhoosh(BaseBloodh ticket_time = ticket.time_changed #act self.req.tz = FixedOffset(60, 'GMT +1:00') - self.req.args["q"] = "*:*" + self.req.args[RequestParameters.QUERY] = "*:*" data = self._process_request() result_items = data["results"].items #asset @@ -135,31 +124,132 @@ class WebUiTestCaseWithWhoosh(BaseBloodh self.assertEqual(expected_datetime, result_datetime) def test_ticket_href(self): - self._insert_docs(1) - self.req.args["q"] = "*:*" + self._insert_tickets(1) + self.req.args[RequestParameters.QUERY] = "*:*" data = self._process_request() docs = data["results"].items self.assertEqual(1, len(docs)) - self.assertEqual(BHSEARCH_URL + "/ticket/1", docs[0]["href"]) + self.assertEqual("/main/ticket/1", docs[0]["href"]) def test_page_href(self): - self._insert_docs(DEFAULT_DOCS_PER_PAGE+1) - self.req.args["q"] = "*:*" + self._insert_tickets(DEFAULT_DOCS_PER_PAGE+1) + self.req.args[RequestParameters.QUERY] = "*:*" data = self._process_request() shown_pages = data["results"].shown_pages - self.assertEqual(BHSEARCH_URL + "/bhsearch?q=*%3A*&page=2&noquickjump=1", shown_pages[1]["href"]) + second_page_href = shown_pages[1]["href"] + self.assertIn("page=2", second_page_href) + self.assertIn("q=*%3A*", second_page_href) - def test_facets(self): + def test_facets_ticket_only(self): self.insert_ticket("summary1 keyword", status="closed") self.insert_ticket("summary2 keyword", status="new") - self.req.args["q"] = "*:*" + self.req.args[RequestParameters.QUERY] = "*:*" data = self._process_request() facets = data["facets"] pprint(facets) - self.assertEqual({u'ticket': 2}, facets["type"]) + self.assertEqual({'ticket': 2}, facets["type"]) + def test_facets_ticket_and_wiki(self): + self.insert_ticket("summary1 keyword", status="closed") + self.insert_ticket("summary2 keyword", status="new") + self.insert_wiki("dummyTitle", "Some text") + self.req.args[RequestParameters.QUERY] = "*" + data = self._process_request() + facets = data["facets"] + pprint(facets) + self.assertEqual({'ticket': 2, 'wiki': 1}, facets["type"]) + + def test_can_apply_type_parameter(self): + #arrange + self.insert_ticket("summary1 keyword", status="closed") + self.insert_ticket("summary2 keyword", status="new") + self.insert_wiki("dummyTitle", "Some text") + self.req.args[RequestParameters.QUERY] = "*" + self.req.args[RequestParameters.TYPE] = "ticket" + #act + data = self._process_request() + docs = data["results"].items + #assert + active_type = data["active_type"] + self.assertEquals("ticket", active_type) + + resource_types = data["types"] + + all = resource_types[0] + self._assertResourceType(all, "All", False) + self.assertNotIn("type", all["href"]) + + ticket = resource_types[1] + self._assertResourceType(ticket, "Ticket", True, "type=ticket") + + wiki = resource_types[2] + self._assertResourceType(wiki, "Wiki", False, "type=wiki") + + def test_type_parameter_in_links(self): + self._insert_tickets(12) + self.req.args[RequestParameters.QUERY] = "*" + self.req.args[RequestParameters.TYPE] = "ticket" + self.req.args[RequestParameters.PAGELEN] = "4" + self.req.args[RequestParameters.PAGE] = "2" + data = self._process_request() + results = data["results"] + docs = results.items + self.assertEquals(4, len(docs)) + + next_chrome_link = self.req.chrome['links']['next'][0]["href"] + self.assertIn('type=ticket', next_chrome_link) + self.assertIn('page=3', next_chrome_link) + + prev_chrome_link = self.req.chrome['links']['prev'][0]["href"] + self.assertIn('type=ticket', prev_chrome_link) + self.assertIn('page=1', prev_chrome_link) + + self.assertIn('type=ticket', data["page_href"]) + + for page in results.shown_pages: + self.assertIn('type=ticket', page["href"]) + + def test_type_grouping(self): + self.req.args[RequestParameters.QUERY] = "*:*" + data = self._process_request() + resource_types = data["types"] + + all = resource_types[0] + self._assertResourceType(all, "All", True) + self.assertNotIn("type", all["href"]) + + ticket = resource_types[1] + self._assertResourceType(ticket, "Ticket", False, "type=ticket") + + wiki = resource_types[2] + self._assertResourceType(wiki, "Wiki", False, "type=wiki") + + + def test_that_there_are_no_page_parameters_for_other_types(self): + #arrange + self._insert_tickets(12) + #act + self.req.args[RequestParameters.QUERY] = "*" + self.req.args[RequestParameters.PAGELEN] = "4" + self.req.args[RequestParameters.PAGE] = "2" + data = self._process_request() + #assert + resource_types = data["types"] + + all = resource_types[0] + self.assertIn("page=2", all["href"]) + + ticket = resource_types[1] + self.assertNotIn("page=", ticket["href"]) + + + def _assertResourceType(self, type, label, active, href_contains = None): + self.assertEquals(label, type["label"]) + self.assertEquals(active, type["active"]) + if href_contains: + self.assertIn(href_contains, type["href"]) - def _insert_docs(self, n): + def _insert_tickets(self, n): for i in range(1, n+1): self.insert_ticket("test %s" % i) def suite(): Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/whoosh_backend.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/whoosh_backend.py?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/whoosh_backend.py (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/whoosh_backend.py Thu Jan 10 13:32:56 2013 @@ -23,20 +23,23 @@ import unittest import tempfile import shutil from bhsearch.api import ASC, DESC, SCORE +from bhsearch.query_parser import DefaultQueryParser from bhsearch.tests.utils import BaseBloodhoundSearchTest from bhsearch.whoosh_backend import WhooshBackend from trac.test import EnvironmentStub from trac.util.datefmt import FixedOffset, utc -from whoosh.qparser import MultifieldParser, MultifieldPlugin, syntax, QueryParser, WhitespacePlugin, PhrasePlugin, PlusMinusPlugin +from whoosh import index, sorting, query +from whoosh.fields import Schema, ID, TEXT, KEYWORD +from whoosh.qparser import MultifieldPlugin, QueryParser, WhitespacePlugin, PhrasePlugin class WhooshBackendTestCase(BaseBloodhoundSearchTest): def setUp(self): self.env = EnvironmentStub(enable=['bhsearch.*']) self.env.path = tempfile.mkdtemp('bhsearch-tempenv') -# self.perm = PermissionSystem(self.env) self.whoosh_backend = WhooshBackend(self.env) self.whoosh_backend.recreate_index() + self.default_parser = DefaultQueryParser(self.env) def tearDown(self): shutil.rmtree(self.env.path) @@ -46,8 +49,7 @@ class WhooshBackendTestCase(BaseBloodhou self.whoosh_backend.add_doc(dict(id="1", type="ticket")) self.whoosh_backend.add_doc(dict(id="2", type="ticket")) result = self.whoosh_backend.query( -# result = self.search_api.query( - "*:*", + query.Every(), sort = [("id", ASC)], ) self.print_result(result) @@ -64,7 +66,7 @@ class WhooshBackendTestCase(BaseBloodhou def test_can_return_all_fields(self): self.whoosh_backend.add_doc(dict(id="1", type="ticket")) - result = self.whoosh_backend.query("*:*") + result = self.whoosh_backend.query(query.Every()) self.print_result(result) docs = result.docs self.assertEqual( @@ -74,7 +76,7 @@ class WhooshBackendTestCase(BaseBloodhou def test_can_select_fields(self): self.whoosh_backend.add_doc(dict(id="1", type="ticket")) - result = self.whoosh_backend.query("*:*", + result = self.whoosh_backend.query(query.Every(), fields=("id", "type")) self.print_result(result) docs = result.docs @@ -87,7 +89,7 @@ class WhooshBackendTestCase(BaseBloodhou self.whoosh_backend.add_doc(dict(id="1", type="ticket")) whoosh_backend2 = WhooshBackend(self.env) whoosh_backend2.add_doc(dict(id="2", type="ticket")) - result = whoosh_backend2.query("*:*") + result = whoosh_backend2.query(query.Every()) self.assertEqual(2, result.hits) def test_can_multi_sort_asc(self): @@ -96,7 +98,7 @@ class WhooshBackendTestCase(BaseBloodhou self.whoosh_backend.add_doc(dict(id="4", type="ticket3")) self.whoosh_backend.add_doc(dict(id="1", type="ticket1")) result = self.whoosh_backend.query( - "*:*", + query.Every(), sort = [("type", ASC), ("id", ASC)], fields=("id", "type"), ) @@ -113,7 +115,7 @@ class WhooshBackendTestCase(BaseBloodhou self.whoosh_backend.add_doc(dict(id="4", type="ticket3")) self.whoosh_backend.add_doc(dict(id="1", type="ticket1")) result = self.whoosh_backend.query( - "*:*", + query.Every(), sort = [("type", ASC), ("id", DESC)], fields=("id", "type"), ) @@ -156,10 +158,12 @@ class WhooshBackendTestCase(BaseBloodhou summary="some text out of search scope", time=the_third_date, )) + + parsed_query = self.default_parser.parse("summary:texttofind") + result = self.whoosh_backend.query( - "summary:texttofind", + parsed_query, sort = [(SCORE, ASC), ("time", DESC)], -# fields=("id", "type"), ) self.print_result(result) self.assertEqual(3, result.hits) @@ -176,7 +180,7 @@ class WhooshBackendTestCase(BaseBloodhou self.whoosh_backend.add_doc(dict(id="2", type="ticket", product="B")) self.whoosh_backend.add_doc(dict(id="3", type="wiki", product="A")) result = self.whoosh_backend.query( - "*:*", + query.Every(), sort = [("type", ASC), ("id", DESC)], fields=("id", "type"), facets= ("type", "product") @@ -187,25 +191,22 @@ class WhooshBackendTestCase(BaseBloodhou self.assertEqual({"ticket":2, "wiki":1}, facets["type"]) self.assertEqual({"A":2, "B":1}, facets["product"]) - @unittest.skip( - "Fix this, check why exception is raise on Whoosh mailing list") - #TODO: fix this!!!! def test_can_do_facet_if_filed_missing_TODO(self): self.whoosh_backend.add_doc(dict(id="1", type="ticket")) self.whoosh_backend.add_doc(dict(id="2", type="ticket", status="New")) result = self.whoosh_backend.query( - "*:*", + query.Every(), facets= ("type", "status") ) self.print_result(result) self.assertEqual(2, result.hits) facets = result.facets self.assertEqual({"ticket":2}, facets["type"]) - self.assertEqual({"new":1}, facets["status"]) + self.assertEqual({None: 1, 'New': 1}, facets["status"]) def test_can_return_empty_result(self): result = self.whoosh_backend.query( - "*:*", + query.Every(), sort = [("type", ASC), ("id", DESC)], fields=("id", "type"), facets= ("type", "product") @@ -216,14 +217,14 @@ class WhooshBackendTestCase(BaseBloodhou def test_can_search_time_with_utc_tzinfo(self): time = datetime(2012, 12, 13, 11, 8, 34, 711957, tzinfo=FixedOffset(0, 'UTC')) self.whoosh_backend.add_doc(dict(id="1", type="ticket", time=time)) - result = self.whoosh_backend.query("*:*") + result = self.whoosh_backend.query(query.Every()) self.print_result(result) self.assertEqual(time, result.docs[0]["time"]) def test_can_search_time_without_tzinfo(self): time = datetime(2012, 12, 13, 11, 8, 34, 711957, tzinfo=None) self.whoosh_backend.add_doc(dict(id="1", type="ticket", time=time)) - result = self.whoosh_backend.query("*:*") + result = self.whoosh_backend.query(query.Every()) self.print_result(result) self.assertEqual(time.replace(tzinfo=utc), result.docs[0]["time"]) @@ -233,60 +234,139 @@ class WhooshBackendTestCase(BaseBloodhou time = datetime(2012, 12, 13, 11, hours, 34, 711957, tzinfo=FixedOffset(tz_diff, "just_one_timezone")) self.whoosh_backend.add_doc(dict(id="1", type="ticket", time=time)) - result = self.whoosh_backend.query("*:*") + result = self.whoosh_backend.query(query.Every()) self.print_result(result) self.assertEqual(datetime(2012, 12, 13, 11, hours-tz_diff, 34, 711957, tzinfo=utc), result.docs[0]["time"]) + + def test_can_apply_filter_and_facet(self): + self.whoosh_backend.add_doc(dict(id="1", type="ticket")) + self.whoosh_backend.add_doc(dict(id="2", type="wiki" )) + result = self.whoosh_backend.query( + query.Every(), + filter=[("type", "ticket")], + facets=["type"] + ) + self.print_result(result) + self.assertEqual(1, result.hits) + self.assertEqual("ticket", result.docs[0]["type"]) + + @unittest.skip("TODO clarify behavior on Whoosh mail list") - def test_can_search_id_and_summary(self): + def test_can_search_id_and_summary_TODO(self): #arrange self.insert_ticket("test x") self.insert_ticket("test 1") -# field_boosts = dict( -# id = 6, -# type = 2, -# summary = 5, -# author = 3, -# milestone = 2, -# keywords = 2, -# component = 2, -# status = 2, -# content = 1, -# changes = 0.8, -# ) fieldboosts = dict( id = 1, summary = 1, ) -# parser = MultifieldParser( -# fieldboosts.keys(), -# WhooshBackend.SCHEMA, -## fieldboosts=field_boosts -# ) - - mfp = MultifieldPlugin(list(fieldboosts.keys()), -# fieldboosts=fieldboosts, -# group=syntax.DisMaxGroup - ) + mfp = MultifieldPlugin(list(fieldboosts.keys()),) pins = [WhitespacePlugin, -# PlusMinusPlugin, PhrasePlugin, mfp] parser = QueryParser(None, WhooshBackend.SCHEMA, plugins=pins) - - parsed_query = parser.parse(u"1") -# parsed_query = parser.parse(u"test") + parsed_query = parser.parse("1") result = self.whoosh_backend.query(parsed_query) self.print_result(result) self.assertEqual(2, result.hits) +class WhooshFunctionalityTestCase(unittest.TestCase): + def setUp(self): + self.index_dir = tempfile.mkdtemp('whoosh_index') + + def tearDown(self): + shutil.rmtree(self.index_dir) + + def test_groupedby_empty_field(self): + schema = Schema( + unique_id=ID(stored=True, unique=True), + id=ID(stored=True), + type=ID(stored=True), + status=KEYWORD(stored=True), + content=TEXT(stored=True), + ) + + ix = index.create_in(self.index_dir, schema=schema) + with ix.writer() as w: + w.add_document(unique_id="1",type="type1") + w.add_document(unique_id="2",type="type2", status="New") + + facet_fields = ("type", "status" ) + groupedby = facet_fields + with ix.searcher() as s: + r = s.search( + query.Every(), + groupedby=groupedby, + maptype=sorting.Count, + ) + facets = self._load_facets(r) + print len(r) == 2 + print facets + self.assertEquals( + {'status': {None: 1, 'New': 1}, 'type': {'type1': 1, 'type2': 1}}, + facets) + + def test_groupedby_empty_field(self): + """ + Whoosh 2.4 raises an error when simultaneously using filters and facets + in search: + AttributeError: 'FacetCollector' object has no attribute 'offset' + + The problem should be fixed in the next release. For more info read + https://bitbucket.org/mchaput/whoosh/issue/274 + + For the time of being, whoosh-backend have to introduce workaround in + order to fix the problem. This unit-test is just a reminder to remove + workaround when the fixed version of Whoosh is applied. + """ + schema = Schema( + unique_id=ID(stored=True, unique=True), + type=ID(stored=True), + ) + + ix = index.create_in(self.index_dir, schema=schema) + with ix.writer() as w: + w.add_document(unique_id=u"1",type=u"type1") + w.add_document(unique_id=u"2",type=u"type2") + + with ix.searcher() as s: + with self.assertRaises(AttributeError): + s.search( + query.Every(), + groupedby=("type"), + maptype=sorting.Count, + filter=query.Term("type", "type1") + ) + +# def _prepare_groupedby(self, facets): +# if not facets: +# return None +# groupedby = sorting.Facets() +# for facet_name in facets: +# groupedby.add_field(facet_name, allow_overlap=True, maptype=sorting.Count) +# return groupedby + + def _load_facets(self, non_paged_results): + facet_names = non_paged_results.facet_names() + if not facet_names: + return None + facets_result = dict() + for name in facet_names: + facets_result[name] = non_paged_results.groups(name) + return facets_result + + def suite(): - return unittest.makeSuite(WhooshBackendTestCase, 'test') + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(WhooshBackendTestCase, 'test')) + suite.addTest(unittest.makeSuite(WhooshFunctionalityTestCase, 'test')) + return suite if __name__ == '__main__': unittest.main() Added: incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/wiki_search.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/wiki_search.py?rev=1431350&view=auto ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/wiki_search.py (added) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/tests/wiki_search.py Thu Jan 10 13:32:56 2013 @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import shutil + +import unittest +import tempfile +from bhsearch.api import BloodhoundSearchApi +from bhsearch.query_parser import DefaultQueryParser +from bhsearch.tests.utils import BaseBloodhoundSearchTest +from bhsearch.whoosh_backend import WhooshBackend +from bhsearch.wiki_search import WikiIndexer, WikiSearchParticipant + +from trac.test import EnvironmentStub +from trac.wiki import WikiSystem, WikiPage + + +class WikiIndexerSilenceOnExceptionTestCase(BaseBloodhoundSearchTest): + def setUp(self): + self.env = EnvironmentStub( + enable=['bhsearch.*'], + path=tempfile.mkdtemp('bhsearch-tempenv'), + ) + self.wiki_indexer = WikiIndexer(self.env) + + def tearDown(self): + pass + + def test_does_not_raise_exception_on_add(self): + self.wiki_indexer.wiki_page_added(None) + + def test_raise_exception_if_configured(self): + self.env.config.set('bhsearch', 'silence_on_error', "False") + self.assertRaises( + Exception, + self.wiki_indexer.wiki_page_added, + None) + +class WikiIndexerEventsTestCase(BaseBloodhoundSearchTest): + DUMMY_PAGE_NAME = "dummyName" + + def setUp(self): + self.env = EnvironmentStub(enable=['bhsearch.*']) + self.env.path = tempfile.mkdtemp('bhsearch-tempenv') + self.env.config.set('bhsearch', 'silence_on_error', "False") + self.wiki_system = WikiSystem(self.env) + self.whoosh_backend = WhooshBackend(self.env) + self.whoosh_backend.recreate_index() + self.search_api = BloodhoundSearchApi(self.env) + self.wiki_participant = WikiSearchParticipant(self.env) + self.query_parser = DefaultQueryParser(self.env) + + def tearDown(self): + shutil.rmtree(self.env.path) + self.env.reset_db() + + def test_can_index_on_new_page(self): + #arrange + self.insert_wiki(self.DUMMY_PAGE_NAME, "dummy text") + #act + results = self.search_api.query("*:*") + #assert + self.print_result(results) + self.assertEqual(1, results.hits) + doc = results.docs[0] + self.assertEqual(self.DUMMY_PAGE_NAME, doc["id"]) + self.assertEqual("dummy text", doc["content"]) + self.assertEqual("wiki", doc["type"]) + + def test_can_delete_page(self): + #arrange + self.insert_wiki(self.DUMMY_PAGE_NAME) + WikiPage(self.env, self.DUMMY_PAGE_NAME).delete() + #act + results = self.search_api.query("*.*") + #assert + self.print_result(results) + self.assertEqual(0, results.hits) + + def test_can_index_changed_event(self): + #arrange + self.insert_wiki(self.DUMMY_PAGE_NAME, "Text to be changed") + page = WikiPage(self.env, self.DUMMY_PAGE_NAME) + page.text = "changed text with keyword" + page.save("anAuthor", "some comment", "::1") + #act + results = self.search_api.query("*:*") + #assert + self.print_result(results) + self.assertEqual(1, results.hits) + doc = results.docs[0] + self.assertEqual("changed text with keyword", doc["content"]) + + def test_can_index_renamed_event(self): + #arrange + self.insert_wiki(self.DUMMY_PAGE_NAME) + page = WikiPage(self.env, self.DUMMY_PAGE_NAME) + page.rename("NewPageName") + #act + results = self.search_api.query("*:*") + #assert + self.print_result(results) + self.assertEqual(1, results.hits) + self.assertEqual("NewPageName", results.docs[0]["id"]) + + def test_can_index_deleted_event(self): + #arrange + self.insert_wiki(self.DUMMY_PAGE_NAME) + page = WikiPage(self.env, self.DUMMY_PAGE_NAME) + page.delete() + #act + results = self.search_api.query("*:*") + #assert + self.print_result(results) + self.assertEqual(0, results.hits) + + def test_can_index_version_deleted_event(self): + #arrange + self.insert_wiki(self.DUMMY_PAGE_NAME, "version1") + page = WikiPage(self.env, self.DUMMY_PAGE_NAME) + page.text = "version 2" + page.save("anAuthor", "some comment", "::1") + page.delete(version=2) + #act + results = self.search_api.query("*:*") + #assert + self.print_result(results) + self.assertEqual(1, results.hits) + self.assertEqual("version1", results.docs[0]["content"]) + + + +def suite(): + suite = unittest.TestSuite() + suite.addTest(unittest.makeSuite(WikiIndexerSilenceOnExceptionTestCase, 'test')) + suite.addTest(unittest.makeSuite(WikiIndexerEventsTestCase, 'test')) + return suite + +if __name__ == '__main__': + unittest.main() Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/ticket_search.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/ticket_search.py?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/ticket_search.py (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/ticket_search.py Thu Jan 10 13:32:56 2013 @@ -19,60 +19,22 @@ # under the License. r"""Ticket specifics for Bloodhound Search plugin.""" -from bhsearch.api import ISearchParticipant, BloodhoundSearchApi +from bhsearch.api import ISearchParticipant, BloodhoundSearchApi, IIndexParticipant, IndexFields from genshi.builder import tag from trac.core import * -from trac.ticket.api import ITicketChangeListener, TicketSystem +from trac.ticket.api import ITicketChangeListener from trac.ticket import Ticket from trac.ticket.query import Query from trac.config import Option -from trac.web.chrome import add_warning -from trac.util.datefmt import to_datetime -TICKET = "ticket" -TICKET_STATUS = 'status' +TICKET_TYPE = "ticket" +TICKET_STATUS = "status" -class TicketSearchParticipant(Component): - implements(ITicketChangeListener, ISearchParticipant) +class TicketIndexer(Component): + implements(ITicketChangeListener, IIndexParticipant) silence_on_error = Option('bhsearch', 'silence_on_error', "True", """If true, do not throw an exception during indexing a resource""") - def _index_ticket(self, ticket, search_api=None, raise_exception = False): - """Internal method for actually indexing a ticket. - This reduces duplicating code.""" - try: - if not search_api: - search_api = BloodhoundSearchApi(self.env) - - #This is very naive prototype implementation - #TODO: a lot of improvements must be added here!!! - contents = { - 'id': unicode(ticket.id), - 'time': ticket.time_changed, - 'type': TICKET, - } - fields = [('component',), ('description','content'), ('component',), - ('keywords',), ('milestone',), ('summary',), - ('status',), ('resolution',), ('reporter','author')] - for f in fields: - if f[0] in ticket.values: - if len(f) == 1: - contents[f[0]] = ticket.values[f[0]] - elif len(f) == 2: - contents[f[1]] = ticket.values[f[0]] - contents['changes'] = u'\n\n'.join([x[4] for x in ticket.get_changelog() - if x[2] == u'comment']) - search_api.add_doc(contents) - except Exception, e: - if (not raise_exception) and self.silence_on_error.lower() == "true": - #Is there any way to get request object to add warning? - # add_warning(req, _('Exception during ticket indexing: %s' % e)) - self.log.error("Error occurs during ticke indexing. \ - The error will not be propagated. Exception: %s", e) - else: - raise - - #ITicketChangeListener methods def ticket_created(self, ticket): """Index a recently created ticket.""" @@ -84,31 +46,82 @@ class TicketSearchParticipant(Component) def ticket_deleted(self, ticket): """Called when a ticket is deleted.""" - s = BloodhoundSearchApi(self.env) - s.delete_doc(u'ticket', unicode(ticket.id)) + try: + search_api = BloodhoundSearchApi(self.env) + search_api.delete_doc(TICKET_TYPE, ticket.id) + except Exception, e: + if self.silence_on_error.lower() == "true": + self.log.error("Error occurs during ticket indexing. \ + The error will not be propagated. Exception: %s", e) + else: + raise - # ISearchParticipant methods - def get_search_filters(self, req=None): - if not req or 'TICKET_VIEW' in req.perm: - return ('ticket', 'Tickets') + def _index_ticket( + self, + ticket, + raise_exception = False, + ): + try: + search_api = BloodhoundSearchApi(self.env) + doc = self.build_doc(ticket) + search_api.add_doc(doc) + except Exception, e: + if (not raise_exception) and self.silence_on_error.lower() == "true": + self.log.error("Error occurs during ticket indexing. \ + The error will not be propagated. Exception: %s", e) + else: + raise - def build_search_index(self, backend): - """ - :type backend: ISearchBackend - """ - #TODO: some king of paging/batch size should be introduced in order to - # avoid loading of all ticket ids in memory - query_records = self.load_tickets_ids() + #IIndexParticipant members + def build_doc(self, trac_doc): + ticket = trac_doc + doc = { + IndexFields.ID: unicode(ticket.id), + IndexFields.TYPE: TICKET_TYPE, + IndexFields.TIME: ticket.time_changed, + } + fields = [('component',), + ('description',IndexFields.CONTENT), + ('keywords',), + ('milestone',), + ('summary',), + ('status',), + ('resolution',), + ('reporter',IndexFields.AUTHOR), + ] + for f in fields: + if f[0] in ticket.values: + if len(f) == 1: + doc[f[0]] = ticket.values[f[0]] + elif len(f) == 2: + doc[f[1]] = ticket.values[f[0]] + doc['changes'] = u'\n\n'.join([x[4] for x in ticket.get_changelog() + if x[2] == u'comment']) + return doc + + def get_entries_for_index(self): + #is there any better way to get all tickets? + query_records = self._load_ticket_ids() for record in query_records: - ticket_id = record["id"] - ticket = Ticket(self.env, ticket_id) - self._index_ticket(ticket, backend, raise_exception=True) + ticket = Ticket(self.env, record["id"]) + yield self.build_doc(ticket) - def load_tickets_ids(self): - #is there better way to get all tickets? + def _load_ticket_ids(self): query = Query(self.env, cols=['id'], order='id') return query.execute() + +class TicketSearchParticipant(Component): + implements(ISearchParticipant) + + #ISearchParticipant members + def get_search_filters(self, req=None): + if not req or 'TICKET_VIEW' in req.perm: + return TICKET_TYPE + + def get_title(self): + return "Ticket" + def format_search_results(self, res): if not TICKET_STATUS in res: stat = 'undefined_status' Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/web_ui.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/web_ui.py?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/web_ui.py (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/web_ui.py Thu Jan 10 13:32:56 2013 @@ -27,16 +27,85 @@ from trac.core import * from genshi.builder import tag from trac.perm import IPermissionRequestor from trac.search import shorten_result +from trac.config import OrderedExtensionsOption from trac.util.presentation import Paginator from trac.util.datefmt import format_datetime, user_time from trac.web import IRequestHandler from trac.util.translation import _ -from trac.web.chrome import (INavigationContributor, ITemplateProvider, - add_link, add_stylesheet, add_warning, - web_context) -from bhsearch.api import BloodhoundSearchApi, ISearchParticipant, SCORE, ASC, DESC +from trac.web.chrome import INavigationContributor, ITemplateProvider, \ + add_link, add_stylesheet +from bhsearch.api import BloodhoundSearchApi, ISearchParticipant, SCORE, ASC, \ + DESC, IndexFields SEARCH_PERMISSION = 'SEARCH_VIEW' +DEFAULT_RESULTS_PER_PAGE = 10 +DEFAULT_SORT = [(SCORE, ASC), ("time", DESC)] + + +class RequestParameters(object): + """ + Helper class for parameter parsing and creation of bhsearch specific URLs. + + Lifecycle of the class must be per request + """ + QUERY = "q" + PAGE = "page" + FILTER = "fl" + TYPE = "type" + NO_QUICK_JUMP = "noquickjump" + PAGELEN = "pagelen" + + def __init__(self, req): + self.req = req + + self.query = req.args.get(RequestParameters.QUERY) + if self.query == None: + self.query = "" + + #TODO: add quick jump functionality + self.noquickjump = 1 + + #TODO: add filters support + self.filters = [] + + #TODO: retrieve sort from query string + self.sort = DEFAULT_SORT + + self.pagelen = int(req.args.get( + RequestParameters.PAGELEN, + DEFAULT_RESULTS_PER_PAGE)) + self.page = int(req.args.get(RequestParameters.PAGE, '1')) + self.type = req.args.get(RequestParameters.TYPE, None) + + self.params = { + self.NO_QUICK_JUMP: self.noquickjump, + } + if self.query: + self.params[self.QUERY] = self.query + if self.pagelen != DEFAULT_RESULTS_PER_PAGE: + self.params[self.PAGELEN]=self.pagelen + if self.page > 1: + self.params[self.PAGE]=self.page + if self.type: + self.params[self.TYPE] = self.type + + def create_href(self, page = None, type=None, skip_type = False, + skip_page = False): + params = dict(self.params) + if page: + params[self.PAGE] = page + + if skip_page and self.PAGE in params: + del(params[self.PAGE]) + + if type: + params[self.TYPE] = type + + if skip_type and self.TYPE in params: + #show all does not require type parameter + del(params[self.TYPE]) + + return self.req.href.bhsearch(**params) class BloodhoundSearchModule(Component): """Main search page""" @@ -46,10 +115,12 @@ class BloodhoundSearchModule(Component): # IWikiSyntaxProvider #todo: implement later ) - search_participants = ExtensionPoint(ISearchParticipant) - - RESULTS_PER_PAGE = 10 - DEFAULT_SORT = [(SCORE, ASC), ("time", DESC)] + search_participants = OrderedExtensionsOption( + 'bhsearch', + 'search_participants', + ISearchParticipant, + "TicketSearchParticipant, WikiSearchParticipant" + ) # INavigationContributor methods @@ -59,119 +130,150 @@ class BloodhoundSearchModule(Component): def get_navigation_items(self, req): if SEARCH_PERMISSION in req.perm: yield ('mainnav', 'bhsearch', - tag.a(_('Bloodhound Search'), href=self.env.href.bhsearch())) + tag.a(_('Bloodhound Search'), href=self.env.href.bhsearch()) + ) # IPermissionRequestor methods def get_permission_actions(self): return [SEARCH_PERMISSION] # IRequestHandler methods - def match_request(self, req): return re.match(r'/bhsearch?', req.path_info) is not None def process_request(self, req): req.perm.assert_permission(SEARCH_PERMISSION) - - query = req.args.get('q') - if query == None: - query = "" + parameters = RequestParameters(req) #TODO add quick jump support - #TODO: refactor filters or replace with facets - filters = [] -# available_filters = filter(None, [p.get_search_filters(req) for p -# in self.search_participants]) -# filters = [f[0] for f in available_filters if req.args.has_key(f[0])] -# if not filters: -# filters = [f[0] for f in available_filters -# if f[0] not in self.default_disabled_filters and -# (len(f) < 3 or len(f) > 2 and f[2])] -# data = {'filters': [{'name': f[0], 'label': f[1], -# 'active': f[0] in filters} -# for f in available_filters], -# 'quickjump': None, -# 'results': []} - + allowed_participants = self._get_allowed_participants(req) data = { - 'query': query, - } + 'query': parameters.query, + } - # Initial page request #todo: filters check, tickets etc - if not any((query, )): + if not any((parameters.query, )): return self._return_data(req, data) - page = int(req.args.get('page', '1')) - - #todo: retrieve sort from query string - sort = self.DEFAULT_SORT + query_filter = self._prepare_query_filter( + parameters.type, + parameters.filters, + allowed_participants) #todo: add proper facets functionality -# facets = ("type", "status") - facets = ("type",) - + facets = self._prepare_facets(req) querySystem = BloodhoundSearchApi(self.env) query_result = querySystem.query( - query, - pagenum = page, - pagelen = self.RESULTS_PER_PAGE, - sort = sort, + parameters.query, + pagenum = parameters.page, + pagelen = parameters.pagelen, + sort = parameters.sort, facets = facets, - ) - ui_docs = [self._process_doc(doc, req) - for doc in query_result.docs] + filter=query_filter) + ui_docs = [self._process_doc(doc, req, allowed_participants) + for doc in query_result.docs] results = Paginator( ui_docs, - page - 1, - self.RESULTS_PER_PAGE, - query_result.hits, - ) + parameters.page - 1, + parameters.pagelen, + query_result.hits) results.shown_pages = self._prepare_shown_pages( - filters, - query, - req, - shown_pages = results.get_shown_pages(self.RESULTS_PER_PAGE)) + parameters, + shown_pages = results.get_shown_pages(parameters.pagelen)) - results.current_page = {'href': None, 'class': 'current', + results.current_page = {'href': None, + 'class': 'current', 'string': str(results.page + 1), 'title':None} if results.has_next_page: - next_href = req.href.bhsearch(zip(filters, ['on'] * len(filters)), - q=req.args.get('q'), page=page + 1, - noquickjump=1) + next_href = parameters.create_href(page = parameters.page + 1) add_link(req, 'next', next_href, _('Next Page')) if results.has_previous_page: - prev_href = req.href.bhsearch(zip(filters, ['on'] * len(filters)), - q=req.args.get('q'), page=page - 1, - noquickjump=1) + prev_href = parameters.create_href(page = parameters.page - 1) add_link(req, 'prev', prev_href, _('Previous Page')) data['results'] = results + self._prepare_type_grouping( + allowed_participants, + parameters, + data) - #add proper facet links + #TODO:add proper facet links data['facets'] = query_result.facets - - data['page_href'] = req.href.bhsearch( - zip(filters, ['on'] * len(filters)), q=req.args.get('q'), - noquickjump=1) + data['page_href'] = parameters.create_href() return self._return_data(req, data) + def _prepare_query_filter(self, type, filters, allowed_participants): + query_filters = [] + + if type in allowed_participants: + query_filters.append((IndexFields.TYPE, type)) + else: + self.log.debug("Unsupported type in web request: %s", type) + + #TODO: handle other filters + return query_filters + + def _prepare_type_grouping(self, allowed_participants, parameters, data): + active_type = parameters.type + if active_type and active_type not in allowed_participants: + raise TracError(_("Unsupported resource type: '%(name)s'", + name=active_type)) + all_is_active = (active_type is None) + grouping = [ + dict( + label=_("All"), + active=all_is_active, + href=parameters.create_href( + skip_type=True, + skip_page=not all_is_active) + ) + ] + + #we want to obtain the same order as specified in search_participants + # option + participant_with_type = dict((participant, type) + for type, participant in allowed_participants.iteritems()) + for participant in self.search_participants: + if participant in participant_with_type: + type = participant_with_type[participant] + is_active = (type == active_type) + grouping.append(dict( + label=_(participant.get_title()), + active=is_active, + href=parameters.create_href( + type=type, + skip_page=not is_active + ) + )) + data["types"] = grouping + data["active_type"] = active_type + + def _prepare_facets(self, req): + facets = [IndexFields.TYPE] + #TODO: add type specific default facets + return facets + + def _get_allowed_participants(self, req): + allowed_participants = {} + for participant in self.search_participants: + type = participant.get_search_filters(req) + if type is not None: + allowed_participants[type] = participant + return allowed_participants + def _return_data(self, req, data): add_stylesheet(req, 'common/css/search.css') return 'bhsearch.html', data, None - def _process_doc(self, doc,req): - titlers = dict([(x.get_search_filters(req)[0], x.format_search_results) - for x in self.search_participants if x.get_search_filters(req)]) - + def _process_doc(self, doc, req, allowed_participants): #todo: introduce copy by predefined value ui_doc = dict(doc) @@ -182,25 +284,25 @@ class BloodhoundSearchModule(Component): if doc.has_key('time'): ui_doc['date'] = user_time(req, format_datetime, doc['time']) - ui_doc['title'] = titlers[doc['type']](doc) + ui_doc['title'] = allowed_participants[doc['type']]\ + .format_search_results(doc) return ui_doc - def _prepare_shown_pages(self, filters, query, req, shown_pages): - pagedata = [] + def _prepare_shown_pages(self, parameters, shown_pages): + page_data = [] for shown_page in shown_pages: - page_href = req.href.bhsearch([(f, 'on') for f in filters], - q=query, - page=shown_page, noquickjump=1) - pagedata.append([page_href, None, str(shown_page), + page_href = parameters.create_href(page=shown_page) + page_data.append([page_href, None, str(shown_page), 'page ' + str(shown_page)]) fields = ['href', 'class', 'string', 'title'] - result_shown_pages = [dict(zip(fields, p)) for p in pagedata] + result_shown_pages = [dict(zip(fields, p)) for p in page_data] return result_shown_pages # ITemplateProvider methods def get_htdocs_dirs(self): -# return [('bhsearch', pkg_resources.resource_filename(__name__, 'htdocs'))] +# return [('bhsearch', +# pkg_resources.resource_filename(__name__, 'htdocs'))] return [] def get_templates_dirs(self): Modified: incubator/bloodhound/trunk/bloodhound_search/bhsearch/whoosh_backend.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/whoosh_backend.py?rev=1431350&r1=1431349&r2=1431350&view=diff ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/whoosh_backend.py (original) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/whoosh_backend.py Thu Jan 10 13:32:56 2013 @@ -22,15 +22,15 @@ r"""Whoosh specific backend for Bloodhou from bhsearch.api import ISearchBackend, DESC, QueryResult, SCORE import os from trac.core import * -from trac.config import Option, PathOption +from trac.config import Option from trac.util.datefmt import utc from whoosh.fields import * -from whoosh import index, sorting -from whoosh.qparser import QueryParser +from whoosh import index, sorting, query from whoosh.searching import ResultsPage -from whoosh.sorting import FieldFacet from whoosh.writing import AsyncWriter -from datetime import datetime, date +from datetime import datetime + +UNIQUE_ID = 'unique_id' class WhooshBackend(Component): """ @@ -68,7 +68,13 @@ class WhooshBackend(Component): self.open_or_create_index_if_missing() #ISearchBackend methods - def add_doc(self, doc, commit=True): + def start_operation(self): + return dict(writer = self._create_writer()) + + def _create_writer(self): + return AsyncWriter(self.index) + + def add_doc(self, doc, writer=None): """Add any type of document index. The contents should be a dict with fields matching the search schema. @@ -76,74 +82,126 @@ class WhooshBackend(Component): """ # Really make sure it's unicode, because Whoosh won't have it any # other way. + is_local_writer = False + if writer is None: + is_local_writer = True + writer = self._create_writer() + for key in doc: doc[key] = self._to_whoosh_format(doc[key]) + doc["unique_id"] = self._create_unique_id(doc["type"], doc["id"]) + self.log.debug("Doc to index: %s", doc) + try: + writer.update_document(**doc) + if is_local_writer: + writer.commit() + except: + if is_local_writer: + writer.cancel() - doc["unique_id"] = u"%s:%s" % (doc["type"], doc["id"]) + def delete_doc(self, type, id, writer=None): + unique_id = self._create_unique_id(type, id) + self.log.debug('Removing document from the index: %s', unique_id) + is_local_writer = False + if writer is None: + is_local_writer = True + writer = self._create_writer() + try: + writer.delete_by_term(UNIQUE_ID, unique_id) + if is_local_writer: + writer.commit() + except: + if is_local_writer: + writer.cancel() + raise + + def optimize(self): writer = AsyncWriter(self.index) - committed = False + writer.commit(optimize=True) + + def commit(self, optimize, writer): + writer.commit(optimize=optimize) + + def cancel(self, writer): try: - #todo: remove it!!! - self.log.debug("Doc to index: %s", doc) - writer.update_document(**doc) - writer.commit() - committed = True - finally: - if not committed: - writer.cancel() + writer.cancel() + except Exception,ex: + self.env.log.error("Error during writer cancellation: %s", ex) + def recreate_index(self): + self.log.info('Creating Whoosh index in %s' % self.index_dir) + self._make_dir_if_not_exists() + return index.create_in(self.index_dir, schema=self.SCHEMA) - def query(self, query, sort = None, fields = None, boost = None, filters = None, + def open_or_create_index_if_missing(self): + if index.exists_in(self.index_dir): + self.index = index.open_dir(self.index_dir) + else: + self.index = self.recreate_index() + + def query(self, query, sort = None, fields = None, boost = None, filter = None, facets = None, pagenum = 1, pagelen = 20): + """ + Perform query. + Temporary fixes: + Whoosh 2.4 raises an error when simultaneously using filters and facets + in search: + AttributeError: 'FacetCollector' object has no attribute 'offset' + The problem should be fixed in the next release. For more info read + https://bitbucket.org/mchaput/whoosh/issue/274 + A workaround is introduced to join query and filter in query and not + using filter parameter. Remove the workaround when the fixed version + of Whoosh is applied. + """ with self.index.searcher() as searcher: - parser = QueryParser("content", self.index.schema) - if isinstance(query, basestring): - query = unicode(query) - parsed_query = parser.parse(unicode(query)) - else: - parsed_query = query - sortedby = self._prepare_sortedby(sort) - groupedby = self._prepare_groupedby(facets) - self.env.log.debug("Whoosh query to execute: %s, sortedby = %s, \ - pagenum=%s, pagelen=%s, facets=%s", - parsed_query, - sortedby, - pagenum, - pagelen, - groupedby, - ) - raw_page = searcher.search_page( - parsed_query, + groupedby = facets + query_filter = self._prepare_filter(filter) + + #workaround of Whoosh bug, read method __doc__ + query = self._workaround_join_query_and_filter( + query, + query_filter) + + search_parameters = dict( + query = query, pagenum = pagenum, pagelen = pagelen, sortedby = sortedby, groupedby = groupedby, + maptype=sorting.Count, + #workaround of Whoosh bug, read method __doc__ + #filter = query_filter, ) -# raw_page = ResultsPage(whoosh_results, pagenum, pagelen) - results = self._process_results(raw_page, fields) + self.env.log.debug("Whoosh query to execute: %s", + search_parameters) + raw_page = searcher.search_page(**search_parameters) + results = self._process_results(raw_page, fields, search_parameters) return results - def delete_doc(self, doc, commit=True): - pass - - def commit(self): - pass - - def optimize(self): - pass + def _workaround_join_query_and_filter( + self, + query_expression, + query_filter): + if not query_filter: + return query_expression + return query.And((query_expression, query_filter)) - def recreate_index(self): - self.index = self._create_index() + def _prepare_filter(self, filters): + if not filters: + return None + and_filters = [] + for filter in filters: + and_filters.append(query.Term( + unicode(filter[0]), + unicode(filter[1]))) + return query.And(and_filters) - def open_or_create_index_if_missing(self): - if index.exists_in(self.index_dir): - self.index = index.open_dir(self.index_dir) - else: - self.index = self._create_index() + def _create_unique_id(self, type, id): + return u"%s:%s" % (type, id) def _to_whoosh_format(self, value): if isinstance(value, basestring): @@ -167,13 +225,13 @@ class WhooshBackend(Component): value = utc.localize(value) return value - def _prepare_groupedby(self, facets): - if not facets: - return None - groupedby = sorting.Facets() - for facet_name in facets: - groupedby.add_field(facet_name, allow_overlap=True, maptype=sorting.Count) - return groupedby +# def _prepare_groupedby(self, facets): +# if not facets: +# return None +# groupedby = sorting.Facets() +# for facet_name in facets: +# groupedby.add_field(facet_name, allow_overlap=True, maptype=sorting.Count) +# return groupedby def _prepare_sortedby(self, sort): if not sort: @@ -194,7 +252,7 @@ class WhooshBackend(Component): def _is_desc(self, order): return (order.lower()==DESC) - def _process_results(self, page, fields): + def _process_results(self, page, fields, search_parameters = None): # It's important to grab the hits first before slicing. Otherwise, this # can cause pagination failures. """ @@ -213,6 +271,7 @@ class WhooshBackend(Component): result_doc = self._process_record(fields, retrieved_record) docs.append(result_doc) results.docs = docs + results.debug["search_parameters"] = search_parameters return results def _process_record(self, fields, retrieved_record): @@ -244,12 +303,7 @@ class WhooshBackend(Component): facets_result[name] = non_paged_results.groups(name) return facets_result - def _create_index(self): - self.log.info('Creating Whoosh index in %s' % self.index_dir) - self._mkdir_if_not_exists() - return index.create_in(self.index_dir, schema=self.SCHEMA) - - def _mkdir_if_not_exists(self): + def _make_dir_if_not_exists(self): if not os.path.exists(self.index_dir): os.mkdir(self.index_dir) Added: incubator/bloodhound/trunk/bloodhound_search/bhsearch/wiki_search.py URL: http://svn.apache.org/viewvc/incubator/bloodhound/trunk/bloodhound_search/bhsearch/wiki_search.py?rev=1431350&view=auto ============================================================================== --- incubator/bloodhound/trunk/bloodhound_search/bhsearch/wiki_search.py (added) +++ incubator/bloodhound/trunk/bloodhound_search/bhsearch/wiki_search.py Thu Jan 10 13:32:56 2013 @@ -0,0 +1,119 @@ +#!/usr/bin/env python +# -*- coding: UTF-8 -*- + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +r"""Wiki specifics for Bloodhound Search plugin.""" +from bhsearch.api import ISearchParticipant, BloodhoundSearchApi, \ + IIndexParticipant, IndexFields +from trac.core import * +from trac.config import Option +from trac.wiki import IWikiChangeListener, WikiSystem, WikiPage + +WIKI_TYPE = "wiki" + +class WikiIndexer(Component): + implements(IWikiChangeListener, IIndexParticipant) + silence_on_error = Option('bhsearch', 'silence_on_error', "True", + """If true, do not throw an exception during indexing a resource""") + + #IWikiChangeListener methods + def wiki_page_added(self, page): + """Index a recently created ticket.""" + self._index_wiki(page) + + def wiki_page_changed(self, page, version, t, comment, author, ipnr): + """Reindex a recently modified ticket.""" + self._index_wiki(page) + + def wiki_page_deleted(self, page): + """Called when a ticket is deleted.""" + try: + search_api = BloodhoundSearchApi(self.env) + search_api.delete_doc(WIKI_TYPE, page.name) + except Exception, e: + if self.silence_on_error.lower() == "true": + self.log.error("Error occurs during wiki indexing. \ + The error will not be propagated. Exception: %s", e) + else: + raise + + def wiki_page_version_deleted(self, page): + """Called when a version of a page has been deleted.""" + self._index_wiki(page) + + def wiki_page_renamed(self, page, old_name): + """Called when a page has been renamed.""" + try: + doc = self.build_doc(page) + search_api = BloodhoundSearchApi(self.env) + search_api.change_doc_id(doc, old_name) + except Exception, e: + if self.silence_on_error.lower() == "true": + self.log.error("Error occurs during wiki indexing. \ + The error will not be propagated. Exception: %s", e) + else: + raise + + def _index_wiki(self, page, raise_exception = False): + try: + doc = self.build_doc(page) + search_api = BloodhoundSearchApi(self.env) + search_api.add_doc(doc) + except Exception, e: + if (not raise_exception) and self.silence_on_error.lower() == "true": + self.log.error("Error occurs during wiki indexing. \ + The error will not be propagated. Exception: %s", e) + else: + raise + + #IIndexParticipant members + def build_doc(self, trac_doc): + page = trac_doc + #This is very naive prototype implementation + #TODO: a lot of improvements must be added here!!! + doc = { + IndexFields.ID: page.name, + IndexFields.TYPE: WIKI_TYPE, + IndexFields.TIME: page.time, + IndexFields.AUTHOR: page.author, + IndexFields.CONTENT: page.text, + } + return doc + + def get_entries_for_index(self): + #is there better way to get all tickets? + page_names = WikiSystem(self.env).get_pages() + for page_name in page_names: + page = WikiPage(self.env, page_name) + yield self.build_doc(page) + +class WikiSearchParticipant(Component): + implements(ISearchParticipant) + + #ISearchParticipant members + def get_search_filters(self, req=None): + if not req or 'WIKI_VIEW' in req.perm: + return WIKI_TYPE + + def get_title(self): + return "Wiki" + + def format_search_results(self, res): + return u'%s: %s...' % (res['id'], res['content'][:50]) +