John Vandenberg has uploaded a new change for review. https://gerrit.wikimedia.org/r/155698
Change subject: WIP: Use ipaddress module ...................................................................... WIP: Use ipaddress module page.py includes a regexp to check if a username is an IPv4 or IPv6 address, for User.isAnonymous(). There are other instances where pywikibot should be validating strings which may contain a IP address, such as config parsing, and a client side assert to prevent writing to the server as an anonymous user. There are modules which provide this functionality, however they fail the pywikibot tests. This changeset doesnt alter the existing tests, so the failures are recorded. Change-Id: I438632608c07f2b7ed196e8b2a536481fcfa3e8e --- M pywikibot/comms/http.py M pywikibot/data/api.py M pywikibot/page.py M setup.py M tests/ipregex_tests.py 5 files changed, 104 insertions(+), 17 deletions(-) git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core refs/changes/98/155698/1 diff --git a/pywikibot/comms/http.py b/pywikibot/comms/http.py index dbc23cf..8da6653 100644 --- a/pywikibot/comms/http.py +++ b/pywikibot/comms/http.py @@ -59,6 +59,14 @@ from http import cookiejar as cookielib from urlparse import quote +try: + # https://pypi.python.org/pypi/py2-ipaddress fails numerous tests + # https://pypi.python.org/pypi/ipaddress only fails a few tests + # _Both_ install as 'ipaddress' + import ipaddress +except ImportError: + ipaddress = None + from pywikibot import config from pywikibot.exceptions import FatalServerError, Server504Error from pywikibot.comms import threadedhttp @@ -124,6 +132,43 @@ pywikibot.cookie_jar = cookie_jar +def is_IP(IP): + """ + Verify the IP address provided is valid. + + @param IP: IP address + @type IP: unicode + @rtype: bool + """ + global ipaddress + if ipaddress: + try: + ipaddress.ip_address(IP) + return True + except ValueError as e: + pywikibot.debug(u'is_IP(%r) failed: %s' + % (IP, e), _logger) + return False + + if ipaddress is None: + # Only print the following warning once. + pywikibot.warning( + u'pywikibot uses the ipaddress module if available.\n' + u'For Python 2, install the backport with pip.') + ipaddress = False + + import re + ip_regexp = re.compile(r'^(?:(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}' + r'(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)|' + r'(((?=(?=(.*?(::)))\3(?!.+\4)))\4?|[\dA-F]{1,4}:)' + r'([\dA-F]{1,4}(\4|:\b)|\2){5}' + r'(([\dA-F]{1,4}(\4|:\b|$)|\2){2}|' + r'(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4}))\Z', + re.IGNORECASE) + + return ip_regexp.match(IP) is not None + + def request(site, uri, ssl=False, *args, **kwargs): """Queue a request to be submitted to Site. diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index 643369c..4e5fcf7 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -146,6 +146,15 @@ "wbremoveclaims", "wbsetclaimvalue", "wbsetreference", "wbremovereferences" ) + # Client side verification that the request is being performed + # by a logged in user. + if self.write: + if not hasattr(self.site, "_userinfo") or not self.site.user(): + raise Error(u"API write action without a user specified") + if http.is_IP(self.site._userinfo['name']): + raise Error(u"API write action as an IP %r" + % self.site._userinfo['name']) + # MediaWiki 1.23 allows assertion for any action, # whereas earlier WMF wikis and others used an extension which # could only allow assert for action=edit. diff --git a/pywikibot/page.py b/pywikibot/page.py index 8c7afe8..c699b9d 100644 --- a/pywikibot/page.py +++ b/pywikibot/page.py @@ -2254,15 +2254,6 @@ return sorted(list(set(self.categories()))) -ip_regexp = re.compile(r'^(?:(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}' - r'(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)|' - r'(((?=(?=(.*?(::)))\3(?!.+\4)))\4?|[\dA-F]{1,4}:)' - r'([\dA-F]{1,4}(\4|:\b)|\2){5}' - r'(([\dA-F]{1,4}(\4|:\b|$)|\2){2}|' - r'(((2[0-4]|1\d|[1-9])?\d|25[0-5])\.?\b){4}))\Z', - re.IGNORECASE) - - class User(Page): """A class that represents a Wiki user. @@ -2338,7 +2329,7 @@ @return: bool """ - return ip_regexp.match(self.username) is not None + return pywikibot.comms.http.is_IP(self.username) def getprops(self, force=False): """ Return a properties about the user. diff --git a/setup.py b/setup.py index 4d64d1f..5524311 100644 --- a/setup.py +++ b/setup.py @@ -43,6 +43,9 @@ testcollector = "tests.utils.collector" dependencies.append('ordereddict') + # https://pypi.python.org/simple/ipaddress + dependencies.append('ipaddress>=1.0.6') + if sys.version_info[0] == 3: if not os.environ.get('PY3', False): # use setup.py test --python3ok to run tests diff --git a/tests/ipregex_tests.py b/tests/ipregex_tests.py index 01fa79b..4db27cf 100644 --- a/tests/ipregex_tests.py +++ b/tests/ipregex_tests.py @@ -6,36 +6,64 @@ # Distributed under the terms of the MIT license. __version__ = '$Id$' +try: + # https://pypi.python.org/pypi/py2-ipaddress fails numerous tests + # https://pypi.python.org/pypi/ipaddress only fails a few tests + # _Both_ install as 'ipaddress' + # The one selected in this test module will be injected into + # the http module to be tested. + import ipaddress +except ImportError as e: + print(e) + ipaddress = False + +from pywikibot.comms import http from tests.utils import unittest, NoSiteTestCase -from pywikibot.page import ip_regexp class PyWikiIpRegexCase(NoSiteTestCase): """Unit test class for ip_regexp""" def setUp(self): + self.mode = None + self.orig_ipaddress = http.ipaddress self.total = 0 self.fail = 0 + self.errors = [] super(PyWikiIpRegexCase, self).setUp() def tearDown(self): super(PyWikiIpRegexCase, self).tearDown() - print '%d tests done, %d failed' % (self.total, self.fail) - if self.fail: - raise AssertionError + http.ipaddress = self.orig_ipaddress + if not self.fail: + print('%d tests done' % self.total) + else: + raise AssertionError( + '%d tests failed:\n%s' + % (self.fail, '\n'.join(self.errors))) def ipv6test(self, result, IP): self.total += 1 failed = False try: - self.assertEqual(result, bool(ip_regexp.match(IP))) + if self.mode == 'regex': + http.ipaddress = False + else: + IP = unicode(IP) + http.ipaddress = ipaddress + + self.assertEqual(result, http.is_IP(IP)) + except AssertionError: self.fail += 1 failed = True if failed: - print '"%s" should match %s - not OK' % (IP, result) + self.errors.append( + 'module=%s "%s" should match %s - not OK' + % (http.ipaddress.__name__ if http.ipaddress else 'None', + IP, result)) - def test_IP(self): + def _run_tests(self): # test from http://download.dartware.com/thirdparty/test-ipv6-regex.pl self.ipv6test(False, "") # empty string self.ipv6test(True, "::1") # loopback, compressed, non-routable @@ -608,6 +636,17 @@ self.ipv6test(True, "a:b:c:d:e:f:0::") self.ipv6test(False, "':10.0.0.1") + def test_regex(self): + self.mode = 'regex' + self._run_tests() + + def test_ipaddress_module(self): + if not ipaddress: + raise unittest.SkipTest('module ipaddress not installed') + + self.mode = 'module' + self._run_tests() + if __name__ == "__main__": try: -- To view, visit https://gerrit.wikimedia.org/r/155698 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I438632608c07f2b7ed196e8b2a536481fcfa3e8e Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jay...@gmail.com> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits