Raoul Snyman has proposed merging lp:~raoul-snyman/openlp/bug-991150 into lp:openlp.
Requested reviews: OpenLP Core (openlp-core) Related bugs: Bug #991150 in OpenLP: "Web Download Bible parse error" https://bugs.launchpad.net/openlp/+bug/991150 For more details, see: https://code.launchpad.net/~raoul-snyman/openlp/bug-991150/+merge/109944 Changed parsing of BG HTML. -- https://code.launchpad.net/~raoul-snyman/openlp/bug-991150/+merge/109944 Your team OpenLP Core is requested to review the proposed merge of lp:~raoul-snyman/openlp/bug-991150 into lp:openlp.
=== modified file 'openlp/plugins/bibles/lib/http.py' --- openlp/plugins/bibles/lib/http.py 2012-04-29 15:31:56 +0000 +++ openlp/plugins/bibles/lib/http.py 2012-06-12 21:32:25 +0000 @@ -43,6 +43,15 @@ from openlp.plugins.bibles.lib.db import BibleDB, BiblesResourcesDB, \ Book +UGLY_CHARS = { + u'\u2014': u' - ', + u'\u2018': u'\'', + u'\u2019': u'\'', + u'\u201c': u'"', + u'\u201d': u'"', + u' ': u' ' +} + log = logging.getLogger(__name__) class BGExtract(object): @@ -54,6 +63,103 @@ self.proxy_url = proxy_url socket.setdefaulttimeout(30) + def _remove_elements(self, parent, tag, class_=None): + """ + Remove a particular element from the BeautifulSoup tree. + + ``parent`` + The element from which items need to be removed. + + ``tag`` + A string of the tab type, e.g. "div" + + ``class_`` + An HTML class attribute for further qualification. + """ + if class_: + all_tags = parent.findAll(tag, class_) + else: + all_tags = parent.findAll(tag) + for element in all_tags: + element.extract() + + def _extract_verse(self, tag): + """ + Extract a verse (or part of a verse) from a tag. + + ``tag`` + The BeautifulSoup Tag element with the stuff we want. + """ + if isinstance(tag, NavigableString): + return None, unicode(tag) + elif tag.get('class') == 'versenum': + verse = unicode(tag.string)\ + .replace('[', '').replace(']', '').strip() + return verse, None + elif tag.get('class') == 'chapternum': + verse = '1' + return verse, None + else: + verse, text = None, '' + for child in tag.contents: + c_verse, c_text = self._extract_verse(child) + if c_verse: + verse = c_verse + if text and c_text: + text += c_text + elif c_text is not None: + text = c_text + return verse, text + + def _clean_soup(self, tag): + """ + Remove all the rubbish from the HTML page. + + ``tag`` + The base tag within which we want to remove stuff. + """ + self._remove_elements(tag, 'sup', 'crossreference') + self._remove_elements(tag, 'sup', 'footnote') + self._remove_elements(tag, 'div', 'footnotes') + self._remove_elements(tag, 'div', 'crossrefs') + self._remove_elements(tag, 'h3') + + def _extract_verses(self, tags): + """ + Extract all the verses from a pre-prepared list of HTML tags. + + ``tags`` + A list of BeautifulSoup Tag elements. + """ + verses = [] + tags = tags[::-1] + current_text = '' + for tag in tags: + verse, text = None, '' + for child in tag.contents: + c_verse, c_text = self._extract_verse(child) + if c_verse: + verse = c_verse + if text and c_text: + text += c_text + elif c_text is not None: + text = c_text + if not verse: + current_text = text + ' ' + current_text + else: + text += ' ' + current_text + current_text = '' + if text: + for old, new in UGLY_CHARS.iteritems(): + text = text.replace(old, new) + text = u' '.join(text.split()) + if verse and text: + verses.append((int(verse.strip()), text)) + verse_list = {} + for verse, text in verses[::-1]: + verse_list[verse] = text + return verse_list + def get_bible_chapter(self, version, book_name, chapter): """ Access and decode Bibles via the BibleGateway website. @@ -80,60 +186,9 @@ if not soup: return None Receiver.send_message(u'openlp_process_events') - footnotes = soup.findAll(u'sup', u'footnote') - if footnotes: - for footnote in footnotes: - footnote.extract() - crossrefs = soup.findAll(u'sup', u'xref') - if crossrefs: - for crossref in crossrefs: - crossref.extract() - headings = soup.findAll(u'h5') - if headings: - for heading in headings: - heading.extract() - chapter_notes = soup.findAll('div', 'footnotes') - if chapter_notes: - log.debug('Found chapter notes') - for note in chapter_notes: - note.extract() - note_comments = soup.findAll(text=u'end of footnotes') - if note_comments: - for comment in note_comments: - comment.extract() - cleanup = [(re.compile('\s+'), lambda match: ' ')] - verses = BeautifulSoup(str(soup), markupMassage=cleanup) - verse_list = {} - # Cater for inconsistent mark up in the first verse of a chapter. - first_verse = verses.find(u'versenum') - if first_verse and first_verse.contents: - verse_list[1] = unicode(first_verse.contents[0]) - for verse in verses(u'sup', u'versenum'): - raw_verse_num = verse.next - clean_verse_num = 0 - # Not all verses exist in all translations and may or may not be - # represented by a verse number. If they are not fine, if they are - # it will probably be in a format that breaks int(). We will then - # have no idea what garbage may be sucked in to the verse text so - # if we do not get a clean int() then ignore the verse completely. - try: - clean_verse_num = int(str(raw_verse_num)) - except ValueError: - log.warn(u'Illegal verse number in %s %s %s:%s', - version, book_name, chapter, unicode(raw_verse_num)) - if clean_verse_num: - verse_text = raw_verse_num.next - part = raw_verse_num.next.next - while not (isinstance(part, Tag) and part.attrMap and - part.attrMap[u'class'] == u'versenum'): - # While we are still in the same verse grab all the text. - if isinstance(part, NavigableString): - verse_text = verse_text + part - if isinstance(part.next, Tag) and part.next.name == u'div': - # Run out of verses so stop. - break - part = part.next - verse_list[clean_verse_num] = unicode(verse_text) + div = soup.find('div', 'result-text-style-normal') + self._clean_soup(div) + verse_list = self._extract_verses(div.findAll('span', 'text')) if not verse_list: log.debug(u'No content found in the BibleGateway response.') send_error_message(u'parse')
_______________________________________________ Mailing list: https://launchpad.net/~openlp-core Post to : openlp-core@lists.launchpad.net Unsubscribe : https://launchpad.net/~openlp-core More help : https://help.launchpad.net/ListHelp