John Vandenberg has uploaded a new change for review. https://gerrit.wikimedia.org/r/150872
Change subject: Introduce static method Link.normalize(title) ...................................................................... Introduce static method Link.normalize(title) Moves normalisation and basic validation logic from Link.__init__ and Link.parse_test into a new static method which can be used on a string which is a title when validation of site namespaces and interwikis is not desirable. Fixes bug 61832 Change-Id: I7021e3d7e40d72fd74709f396e967c9803248c52 --- M pywikibot/page.py M pywikibot/site.py M scripts/cosmetic_changes.py M tests/wikibase_tests.py 4 files changed, 92 insertions(+), 43 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core refs/changes/72/150872/1 diff --git a/pywikibot/page.py b/pywikibot/page.py index 7a0d1bd..b17cded 100644 --- a/pywikibot/page.py +++ b/pywikibot/page.py @@ -1234,12 +1234,15 @@ # element into a list in the format used by old scripts result = [] for template in templates: - link = pywikibot.Link(template[0], self.site, - defaultNamespace=10) try: + link = pywikibot.Link(template[0], self.site, + defaultNamespace=10) if link.canonical_title() not in titles: continue - except pywikibot.Error: + except pywikibot.InvalidTitle: + # TODO: this exception handling should not be necessary, + # however extract_templates_and_params on en:wp mainpage + # returns titles like '#if:{{Main Page banner}}'. # this is a parser function or magic word, not template name continue args = template[1] @@ -3505,10 +3508,14 @@ following a '|' character inside the link """ - + # The hash/number symbol and vertical pipe symbol are valid in a Link, + # but are not valid in a Title. illegal_titles_pattern = re.compile( + r'''[\x23\x7c]''' + ) + illegal_link_pattern = re.compile( # Matching titles will be held as illegal. - r'''[\x00-\x1f\x23\x3c\x3e\x5b\x5d\x7b\x7c\x7d\x7f]''' + r'''[\x00-\x1f\x3c\x3e\x5b\x5d\x7b\x7d\x7f]''' # URL percent encoding sequences interfere with the ability # to round-trip titles -- you can't link to them consistently. u'|%[0-9A-Fa-f]{2}' @@ -3531,27 +3538,40 @@ contain one (defaults to 0) @type defaultNamespace: int + @exception InvalidTitle: The title is not valid. """ assert source is None or isinstance(source, pywikibot.site.BaseSite), \ "source parameter should be a Site object" + + if not text: + raise pywikibot.InvalidTitle('No title') self._text = text self._source = source or pywikibot.Site() self._defaultns = defaultNamespace - # preprocess text (these changes aren't site-dependent) - # First remove anchor, which is stored unchanged, if there is one - if u"|" in self._text: - self._text, self._anchor = self._text.split(u"|", 1) - else: - self._anchor = None - # Clean up the name, it can come from anywhere. - # Convert HTML entities to unicode - t = html2unicode(self._text) # Convert URL-encoded characters to unicode - t = url2unicode(t, site=self._source) + # FIXME: to be moved into normalize after + # I9ca2a933d227afa79de8ce402304592682785d17 + t = url2unicode(self._text, site=self._source) + + self._text = Link.normalize(t) + + @staticmethod + def normalize(title): + """ + Normalise a title, with basic non-site specific validation. + + @param title: title to normalise + @type title: unicode + @return: unicode + + @exception InvalidTitle: The title is not valid. + """ + # Convert HTML entities to unicode + t = html2unicode(title) # Normalize unicode string to a NFC (composed) format to allow # proper string comparisons. According to @@ -3563,8 +3583,8 @@ # This code was adapted from Title.php : secureAndSplit() # if u'\ufffd' in t: - raise pywikibot.Error( - "Title contains illegal char (\\uFFFD 'REPLACEMENT CHARACTER')") + raise pywikibot.InvalidTitle( + "Title contains illegal char \\uFFFD (REPLACEMENT CHARACTER)") # Replace underscores by spaces t = t.replace(u"_", u" ") @@ -3572,10 +3592,41 @@ while u" " in t: t = t.replace(u" ", u" ") # Strip spaces at both ends + # TODO: Stripping trailing spaces breaks linktrails + # and may cause the same issue with leading spaces. t = t.strip() # Remove left-to-right and right-to-left markers. t = t.replace(u"\u200e", u"").replace(u"\u200f", u"") - self._text = t + + # Reject illegal characters. + m = Link.illegal_link_pattern.search(t) + if m: + raise pywikibot.InvalidTitle( + u"%s contains illegal char(s) %s" + % (repr(t), repr(m.group(0)))) + + # Pages with "/./" or "/../" appearing in the URLs will + # often be unreachable due to the way web browsers deal + # * with 'relative' URLs. Forbid them explicitly. + + if u'.' in t and ( + t == u'.' or t == u'..' + or t.startswith(u"./") + or t.startswith(u"../") + or u"/./" in t + or u"/../" in t + or t.endswith(u"/.") + or t.endswith(u"/..") + ): + raise pywikibot.InvalidTitle( + "(contains . / combinations): '%s'" + % self._text) + + # Magic tilde sequences? Nu-uh! + if u"~~~" in t: + raise pywikibot.InvalidTitle("(contains ~~~): '%s'" % self._text) + + return t def __repr__(self): """Return a more complete string representation.""" @@ -3628,6 +3679,14 @@ """ self._site = self._source self._namespace = self._defaultns + + # preprocess text (these changes aren't site-dependent) + # First remove anchor, which is stored unchanged, if there is one + if u"|" in self._text: + self._text, self._anchor = self._text.split(u"|", 1) + else: + self._anchor = None + t = self._text # This code was adapted from Title.php : secureAndSplit() @@ -3698,27 +3757,6 @@ if m: raise pywikibot.InvalidTitle( u"%s contains illegal char(s) %s" % (repr(t), repr(m.group(0)))) - - # Pages with "/./" or "/../" appearing in the URLs will - # often be unreachable due to the way web browsers deal - # * with 'relative' URLs. Forbid them explicitly. - - if u'.' in t and ( - t == u'.' or t == u'..' - or t.startswith(u"./") - or t.startswith(u"../") - or u"/./" in t - or u"/../" in t - or t.endswith(u"/.") - or t.endswith(u"/..") - ): - raise pywikibot.InvalidTitle( - "(contains . / combinations): '%s'" - % self._text) - - # Magic tilde sequences? Nu-uh! - if u"~~~" in t: - raise pywikibot.InvalidTitle("(contains ~~~): '%s'" % self._text) if self._namespace != -1 and len(t) > 255: raise pywikibot.InvalidTitle("(over 255 bytes): '%s'" % t) diff --git a/pywikibot/site.py b/pywikibot/site.py index c26e8cd..b00c16c 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -171,7 +171,12 @@ user = user[0].upper() + user[1:] if sysop: sysop = sysop[0].upper() + sysop[1:] + if user: + user = pywikibot.Link.normalize(user) + if sysop: + sysop = pywikibot.Link.normalize(sysop) self._username = [user, sysop] + self.use_hard_category_redirects = ( self.code in self.family.use_hard_category_redirects) @@ -424,6 +429,14 @@ if title1 == title2: return True + try: + title1 = pywikibot.Link.normalize(title1) + title2 = pywikibot.Link.normalize(title2) + except InvalidTitle: + return False + if title1 == title2: + return True + # determine whether titles contain namespace prefixes if ":" in title1: ns1, name1 = title1.split(":", 1) diff --git a/scripts/cosmetic_changes.py b/scripts/cosmetic_changes.py index 34a2d4f..a305c31 100755 --- a/scripts/cosmetic_changes.py +++ b/scripts/cosmetic_changes.py @@ -405,6 +405,8 @@ except pywikibot.InvalidTitle: return match.group() if namespace == 0: + # TODO: This logic could be merged with Link.normalize + # Replace underlines by spaces, also multiple underlines titleWithSection = re.sub('_+', ' ', titleWithSection) # Remove double spaces diff --git a/tests/wikibase_tests.py b/tests/wikibase_tests.py index aca60b3..a76457e 100644 --- a/tests/wikibase_tests.py +++ b/tests/wikibase_tests.py @@ -289,10 +289,6 @@ self.assertEquals(hasattr(item, '_content'), True) self.assertEquals(item.exists(), True) - def test_fromPage_invalid_title(self): - page = pywikibot.Page(pywikibot.page.Link("[]", site)) - self.assertRaises(pywikibot.InvalidTitle, pywikibot.ItemPage.fromPage, page) - def _test_fromPage_noitem(self, link): """Helper function to test a page without an associated item. -- To view, visit https://gerrit.wikimedia.org/r/150872 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I7021e3d7e40d72fd74709f396e967c9803248c52 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jay...@gmail.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits