I implemented some of the methods in ISearchBackend for Solr, however some of them (especially query()) need more customisation, so I’ll work on that next.
On 14 June 2014 at 20:15:01, [email protected] ([email protected]) wrote: Author: ahorincar Date: Sat Jun 14 19:14:33 2014 New Revision: 1602620 URL: http://svn.apache.org/r1602620 Log: Implemented methods from ISearchBackend for Solr Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/htdocs/ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/__init__.py bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/changeset_search.py bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/milestone_search.py bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/ticket_search.py bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/wiki_search.py bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/templates/ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/templates/bh_solr_test.html bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/web_ui.py Removed: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/index.py Modified: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/schemadoc/schema.xml bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr.py bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py Modified: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/schemadoc/schema.xml URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/schemadoc/schema.xml?rev=1602620&r1=1602619&r2=1602620&view=diff ============================================================================== --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/schemadoc/schema.xml (original) +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/schemadoc/schema.xml Sat Jun 14 19:14:33 2014 @@ -8,39 +8,36 @@ <field name="_root_" type="string" indexed="true" stored="false"/> <!-- BH fields --> - <field name="unique_id" type="string" indexed="true" stored="true" required="true" multiValued="false" /> <field name="id" type="string" indexed="true" stored="true" required="true" multiValued="false" /> + <field name="unique_id" type="string" indexed="true" stored="true" required="true" multiValued="false" /> <field name="type" type="string" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="product" type="string" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="milestone" type="string" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="time" type="date" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="due" type="date" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="completed" type="date" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="author" type="string" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="component" type="string" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="status" type="string" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="resolution" type="string" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="keywords" type="string" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="summary" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="content" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="changes" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="owner" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="repository" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="revision" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="message" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="required_permission" type="string" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="name" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="query_suggestion_basket" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/> - <field name="relations" type="lowercase" indexed="true" stored="true" required="true" multiValued="false"/> - <field/> - - - <field name="name" type="text_general" indexed="true" stored="true"/> + <field name="product" type="string" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="milestone" type="string" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="time" type="date" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="due" type="date" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="completed" type="date" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="author" type="string" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="component" type="string" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="status" type="string" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="resolution" type="string" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="keywords" type="string" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="summary" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="content" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="changes" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="owner" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="repository" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="revision" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="message" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="required_permission" type="string" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="name" type="text_general" indexed="true" stored="true" required="false" multiValued="false"/> + <field name="_stored_name" type="string" indexed="true" stored="true" required="false" multiValued="false"/> + <!-- <field name="query_suggestion_basket" type="text_general" indexed="true" stored="true" required="true" multiValued="false"/> --> + <!-- <field name="relations" type="lowercase" indexed="true" stored="true" required="true" multiValued="false"/> --> </fields> <uniqueKey>unique_id</uniqueKey> -<copyField source="name" dest="text"/> +<!-- <copyField source="name" dest="text"/> --> <types> <!-- Field type definitions --> @@ -60,13 +57,16 @@ <filter class="solr.LowerCaseFilterFactory"/> </analyzer> </fieldType> + <fieldType name="date" class="solr.TrieDateField" precisionStep="0" positionIncrementGap="0"/> + <fieldType name="lowercase" class="solr.TextField" positionIncrementGap="100"> - <analyzer> - <tokenizer class="solr.KeywordTokenizerFactory"/> - <filter class="solr.LowerCaseFilterFactory" /> - </analyzer> -</fieldType> + <analyzer> + <tokenizer class="solr.KeywordTokenizerFactory"/> + <filter class="solr.LowerCaseFilterFactory" /> + </analyzer> + </fieldType> +</types> </schema> Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/__init__.py URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/__init__.py?rev=1602620&view=auto ============================================================================== (empty) Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/changeset_search.py URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/changeset_search.py?rev=1602620&view=auto ============================================================================== --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/changeset_search.py (added) +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/changeset_search.py Sat Jun 14 19:14:33 2014 @@ -0,0 +1,17 @@ +from bhsearch.search_resources.base import BaseIndexer +from trac.versioncontrol.api import RepositoryManager +from bhsearch.search_resources.changeset_search import ChangesetIndexer + +class ChangesetSearchModel(BaseIndexer): + + def get_entries_for_index(self): + repository_manager = RepositoryManager(self.env) + for repository in repository_manager.get_real_repositories(): + rev = repository.oldest_rev + stop = repository.youngest_rev + while True: + changeset = repository.get_changeset(rev) + yield ChangesetIndexer(self.env).build_doc(changeset) + if rev == stop: + break + rev = repository.next_rev(rev) Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/milestone_search.py URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/milestone_search.py?rev=1602620&view=auto ============================================================================== --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/milestone_search.py (added) +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/milestone_search.py Sat Jun 14 19:14:33 2014 @@ -0,0 +1,9 @@ +from bhsearch.search_resources.base import BaseIndexer +from trac.ticket import Milestone +from bhsearch.search_resources.milestone_search import MilestoneIndexer + +class MilestoneSearchModel(BaseIndexer): + + def get_entries_for_index(self): + for milestone in Milestone.select(self.env, include_completed=True): + yield MilestoneIndexer(self.env).build_doc(milestone) Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/ticket_search.py URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/ticket_search.py?rev=1602620&view=auto ============================================================================== --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/ticket_search.py (added) +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/ticket_search.py Sat Jun 14 19:14:33 2014 @@ -0,0 +1,27 @@ +from trac.ticket.model import Ticket +from bhsearch.search_resources.ticket_search import TicketIndexer +from trac.core import Component, implements, TracError +from bhsearch.search_resources.base import BaseIndexer + +class TicketSearchModel(BaseIndexer): + + def _fetch_tickets(self, **kwargs): + for ticket_id in self._fetch_ids(**kwargs): + yield Ticket(self.env, ticket_id) + + def _fetch_ids(self, **kwargs): + sql = "SELECT id FROM ticket" + args = [] + conditions = [] + for key, value in kwargs.iteritems(): + args.append(value) + conditions.append(key + "=%s") + if conditions: + sql = sql + " WHERE " + " AND ".join(conditions) + for row in self.env.db_query(sql, args): + yield int(row[0]) + + def get_entries_for_index(self): + for ticket in self._fetch_tickets(): + yield TicketIndexer(self.env).build_doc(ticket) + Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/wiki_search.py URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/wiki_search.py?rev=1602620&view=auto ============================================================================== --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/wiki_search.py (added) +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/search_resources/wiki_search.py Sat Jun 14 19:14:33 2014 @@ -0,0 +1,11 @@ +from trac.wiki import WikiSystem, WikiPage +from bhsearch.search_resources.wiki_search import WikiIndexer +from bhsearch.search_resources.base import BaseIndexer + +class WikiSearchModel(BaseIndexer): + + def get_entries_for_index(self): + page_names = WikiSystem(self.env).get_pages() + for page_name in page_names: + page = WikiPage(self.env, page_name) + yield WikiIndexer(self.env).build_doc(page) Modified: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr.py URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr.py?rev=1602620&r1=1602619&r2=1602620&view=diff ============================================================================== --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr.py (original) +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr.py Sat Jun 14 19:14:33 2014 @@ -3,8 +3,8 @@ from httplib2 import Http class Solr(): - def __init__(self, solr_url, schema_file): - """ Creates a SolrInterface object with the solr server url and a custom schema + def __init__(self, solr_url): + """ Creates a SolrInterface object with the solr server url and a custom schema file""" self.solr_url = solr_url - self.solr_interface = SolrInterface(url=solr_url, schemadoc=schema_file) + self.solr_interface = SolrInterface(url=solr_url) Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py?rev=1602620&view=auto ============================================================================== --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py (added) +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/solr_backend.py Sat Jun 14 19:14:33 2014 @@ -0,0 +1,86 @@ +import pkg_resources + +from solr import Solr +from trac.ticket.model import Ticket +from trac.core import Component, implements, TracError +from trac.ticket.api import TicketSystem +from bhsearch.search_resources.ticket_search import TicketIndexer +from datetime import datetime +from trac.util.datefmt import utc + +UNIQUE_ID = "unique_id" + +class SolrModel(Component): + implements(ISearchBackend) + + def __init__(self): + resource_filename = pkg_resources.resource_filename + path = resource_filename(__name__, "schemadoc") + # self.solr_interface = Solr("http://localhost:8983/solr/";, path + '/schema.xml').solr_interface + self.solr_interface = Solr("http://localhost:8983/solr/";).solr_interface + + + def add_doc(self, doc, operation_context=None): + self._reformat_doc(doc) + doc[UNIQUE_ID] = self._create_unique_id(doc.get("product", ''), + doc["type"], + doc["id"]) + self.solr_interface.add(doc) + self.solr_interface.commit() + + + def delete_doc(product, doc_type, doc_id, operation_context=None): + unique_id = self._create_unique_id(product, doc_type, doc_id) + self.solr_interface.delete(unique_id) + + + def optimize(): + self.solr_interface.optimize() + + + def query(self, query): + self.solr_instance.solr_interface.query(query).execute() + + + def _reformat_doc(self, doc): + for key, value in doc.items(): + if key is None: + del doc[None] + elif value is None: + del doc[key] + elif isinstance(value, basestring) and value == "": + del doc[key] + else: + doc[key] = self._to_solr_format(value) + + + def _to_solr_format(self, value): + if isinstance(value, basestring): + value = unicode(value) + elif isinstance(value, datetime): + value = self._convert_date_to_tz_naive_utc(value) + return value + + + def _convert_date_to_tz_naive_utc(self, value): + if value.tzinfo: + utc_time = value.astimezone(utc) + value = utc_time.replace(tzinfo=None) + return value + + + def _create_unique_id(self, product, doc_type, doc_id): + if product: + return u"%s:%s:%s" % (product, doc_type, doc_id) + else: + return u"%s:%s" % (doc_type, doc_id) + +if __name__ == '__main__': + env = trac.env.Environment("/Users/antonia/Documents/Code/bloodhound/installer/bloodhound/environments/main") + db_connection = env.get_db_cnx() + cursor = db_connection.cursor() + a = cursor.execute("select * from ticket") + ticket = a.fetchall()[0] + + # for result in si.query(name="Ticket").execute(): + # print result Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/templates/bh_solr_test.html URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/templates/bh_solr_test.html?rev=1602620&view=auto ============================================================================== --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/templates/bh_solr_test.html (added) +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/templates/bh_solr_test.html Sat Jun 14 19:14:33 2014 @@ -0,0 +1,14 @@ +<!DOCTYPE html + PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" + "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd";> +<html xmlns="http://www.w3.org/1999/xhtml"; + xmlns:xi="http://www.w3.org/2001/XInclude"; + xmlns:py="http://genshi.edgewall.org/"; + xmlns:i18n="http://genshi.edgewall.org/i18n";> + <link rel="stylesheet" href="${href.chrome('dashboard/css/bootstrap.css')}" type="text/css" /> + <link rel="stylesheet" href="${href.chrome('dashboard/css/bootstrap-responsive.css')}" type="text/css" /> + + <head> + <title>Test</title> + </head> +</html> Added: bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/web_ui.py URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/web_ui.py?rev=1602620&view=auto ============================================================================== --- bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/web_ui.py (added) +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/bhsolr/web_ui.py Sat Jun 14 19:14:33 2014 @@ -0,0 +1,42 @@ +import re +import os +import pkg_resources + +from trac.web.main import IRequestHandler +from trac.core import Component, TracError, implements +from trac.ticket.model import Ticket +from trac.web.chrome import ITemplateProvider +from bhsolr.search_resources.ticket_search import TicketSearchModel +from bhsolr.search_resources.milestone_search import MilestoneSearchModel +from bhsolr.search_resources.changeset_search import ChangesetSearchModel +from bhsolr.search_resources.wiki_search import WikiSearchModel +from bhsolr.solr_backend import SolrModel + +class BloodhoundSolrSearchModule(Component): + implements(IRequestHandler, ITemplateProvider) + + def match_request(self, req): + if re.match(r'/solr$', req.path_info): + return True + + def process_request(self, req): + # changeset_doc = next(ChangesetSearchModel(self.env).get_entries_for_index()) + milestone_doc = next(MilestoneSearchModel(self.env).get_entries_for_index()) + ticket_doc = next(TicketSearchModel(self.env).get_entries_for_index()) + wiki_doc = next(WikiSearchModel(self.env).get_entries_for_index()) + + SolrModel(self.env).addDoc(ticket_doc) + SolrModel(self.env).addDoc(milestone_doc) + SolrModel(self.env).addDoc(wiki_doc) + + data = {} + return 'bh_solr_test.html', data, None + + def get_templates_dirs(self): + resource_filename = pkg_resources.resource_filename + return [resource_filename('bhsolr', 'templates')] + + def get_htdocs_dirs(self): + resource_filename = pkg_resources.resource_filename + return [('solr', resource_filename('bhsolr', 'htdocs'))] + Modified: bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py URL: http://svn.apache.org/viewvc/bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py?rev=1602620&r1=1602619&r2=1602620&view=diff ============================================================================== --- bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py (original) +++ bloodhound/branches/bep_0014_solr/bloodhound_solr/setup.py Sat Jun 14 19:14:33 2014 @@ -1,17 +1,34 @@ -"""setup for embeddable objects plugin""" -from setuptools import setup +from setuptools import setup, find_packages + +PKG_INFO = {'bhsolr': ['htdocs/*.*', 'templates/*', 'schemadoc/*.xml'], + 'bhsolr.search_resources' : [], + } -setup( - name = 'BloodhoundSolrPlugin', - version = '0.1', - description = "Apache Solr support for Apache(TM) Bloodhound.", - author = "Apache Bloodhound", - license = "Apache License v2", - url = "http://bloodhound.apache.org/";, - packages = ['bhsolr',], - package_data = {'bhsolr' : []}, - entry_points = {'trac.plugins': ['bhsolr.index = bhsolr.index'],}, - test_suite='bhsorl.tests.test_suite', -) +ENTRY_POINTS = { + 'trac.plugins': [ + 'bhsolr.web_ui = bhsolr.web_ui', + 'bhsolr.api = bhsolr.api', + 'bhsolr.solr = bhsolr.solr', + 'bhsolr.solr_backend = bhsolr.solr_backend', + 'bhsolr.search_resources.ticket_search = bhsolr.search_resources.ticket_search', + 'bhsolr.search_resources.milestone_search = bhsolr.search_resources.milestone_search', + 'bhsolr.search_resources.changeset_search = bhsolr.search_resources.changeset_search', + 'bhsolr.search_resources.wiki_search = bhsolr.search_resources.wiki_search' + ],} + +setup( + name = 'BloodhoundSolrPlugin', + version = '0.1', + description = "Apache Solr support for Apache(TM) Bloodhound.", + author = "Apache Bloodhound", + license = "Apache License v2", + url = "http://bloodhound.apache.org/";, + # package_dir = PKG_INFO, + packages = find_packages(), + package_data = PKG_INFO, + include_package_data=True, + entry_points = ENTRY_POINTS, + test_suite='bhsolr.tests.test_suite', +)
