jenkins-bot has submitted this change and it was merged. Change subject: [FEAT] Request: Separate MW API and PWB API params ......................................................................
[FEAT] Request: Separate MW API and PWB API params Instead of using kwargs, which could clash with already existing normal parameters it uses a separate argument which takes all the parameters used for the API. To support the mixed mode in other classes forwarding kwargs directly to Request they can use 'clean_kwargs' which separates the API parameters and Python arguments to use the new style. Subsequent changes must reflect that then. This will not break backwards compatibility, just that implementations not using 'clean_kwargs' will only be able to use one mode. It also adds support for Page instances in parameters and adds a function in tools to merge multiple dicts if they have unique keys. The APISite instance also gets '_request', '_simple_request' and '_request_class' methods in case a script want to use their subclass of Request. The QueryGenerator also uses '_request' and '_request_class'. But for backwards compatibility it allows both modes (so it's converting kwargs mode to parameters mode). With that QueryGenerator uses CachedRequest when an expiry parameter is set which breaks any usage with such an API parameter unless they use the parameters mode. Bug: T70988 Change-Id: Ic259420c312d4efe4f14c772b369f86e6ed2e819 --- M pywikibot/data/api.py M pywikibot/page.py M pywikibot/site.py M pywikibot/tools/__init__.py M scripts/casechecker.py M scripts/redirect.py M scripts/revertbot.py M tests/__init__.py M tests/api_tests.py M tests/tools_tests.py M tests/utils.py 11 files changed, 486 insertions(+), 213 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index bec26a6..43daab3 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -14,6 +14,7 @@ from email.mime.nonmultipart import MIMENonMultipart import datetime import hashlib +import inspect import json import os try: @@ -315,8 +316,9 @@ # paraminfo only exists 1.12+. # Request need ParamInfo to determine use_get - request = CachedRequest(expiry=config.API_config_expiry, - use_get=True, site=self.site, action='help') + request = self.site._request(expiry=config.API_config_expiry, + use_get=True, + parameters={'action': 'help'}) result = request.submit() assert('help' in result) @@ -564,9 +566,6 @@ self.preloaded_modules |= set(['query']) params = { - 'expiry': config.API_config_expiry, - 'use_get': True, # Request need ParamInfo to determine use_get - 'site': self.site, 'action': 'paraminfo', } @@ -582,7 +581,10 @@ for mod in set(module_batch) & self.root_modules: params[mod + 'module'] = 1 - request = CachedRequest(**params) + # Request need ParamInfo to determine use_get + request = self.site._request(expiry=config.API_config_expiry, + use_get=True, + parameters=params) result = request.submit() normalized_result = self.normalize_paraminfo(result) @@ -1063,7 +1065,7 @@ Example: - >>> r = Request(action="query", meta="userinfo") + >>> r = Request(parameters={'action': 'query', 'meta': 'userinfo'}) >>> # This is equivalent to >>> # https://{path}/api.php?action=query&meta=userinfo&format=json >>> # change a parameter @@ -1093,47 +1095,103 @@ """ - def __init__(self, **kwargs): - """ - Constructor. + # To make sure the default value of 'parameters' can be identified. + _PARAM_DEFAULT = object() - @kwarg site: The Site to which the request will be submitted. If not + def __init__(self, site=None, mime=None, throttle=True, mime_params=None, + max_retries=None, retry_wait=None, use_get=None, + parameters=_PARAM_DEFAULT, **kwargs): + """ + Create a new Request instance with the given parameters. + + The parameters for the request can be defined via eiter the 'parameters' + parameter or the keyword arguments. The keyword arguments were the + previous implementation but could cause problems when there are + arguments to the API named the same as normal arguments to this class. + So the second parameter 'parameters' was added which just contains all + parameters. When a Request instance is created it must use either one + of them and not both at the same time. To have backwards compatibility + it adds a parameter named 'parameters' to kwargs when both parameters + are set as that indicates an old call and 'parameters' was originally + supplied as a keyword parameter. + + If undefined keyword arguments were given AND the 'parameters' + parameter was supplied as a positional parameter it still assumes + 'parameters' were part of the keyword arguments. + + If a class is using Request and is directly forwarding the parameters, + L{Request.clean_kwargs} can be used to automatically convert the old + kwargs mode into the new parameter mode. This normalizes the arguments + so that when the API parameters are modified the changes can always be + applied to the 'parameters' parameter. + + @param parameters: The parameters used for the request to the API. + @type parameters: dict + @param site: The Site to which the request will be submitted. If not supplied, uses the user's configured default Site. - @kwarg mime: If true, send in "multipart/form-data" format (default False) - @kwarg mime_params: A dictionary of parameter which should only be - transferred via mime mode. If not None sets mime to True. - @kwarg max_retries: (optional) Maximum number of times to retry after + @param mime: If true, send in "multipart/form-data" format (default + False). Parameters which should only be transferred via mime mode + can be defined via that parameter too (an empty dict acts like + 'True' not like 'False'!). + @type mime: bool or dict + @param mime_params: DEPRECATED! A dictionary of parameter which should + only be transferred via mime mode. If not None sets mime to True. + @param max_retries: (optional) Maximum number of times to retry after errors, defaults to 25 - @kwarg retry_wait: (optional) Minimum time to wait after an error, + @param retry_wait: (optional) Minimum time to wait after an error, defaults to 5 seconds (doubles each retry until max of 120 is reached) - @kwarg use_get: (optional) Use HTTP GET request if possible. If False + @param use_get: (optional) Use HTTP GET request if possible. If False it uses a POST request. If None, it'll try to determine via action=paraminfo if the action requires a POST. - @kwarg format: (optional) Defaults to "json" + @param kwargs: The parameters used for the request to the API. """ - try: - self.site = kwargs.pop("site") - except KeyError: + if site is None: self.site = pywikibot.Site() warn('Request() invoked without a site', RuntimeWarning, 2) - if 'mime_params' in kwargs: - self.mime_params = kwargs.pop('mime_params') + else: + self.site = site + if mime_params is not None: # mime may not be different from mime_params - if 'mime' in kwargs and kwargs.pop('mime') != self.mime: + if mime is not None and mime is not True: raise ValueError('If mime_params is set, mime may not differ ' 'from it.') + warn('Use the "mime" parameter instead of the "mime_params".') + mime = mime_params + self.mime = mime # this also sets self.mime_params + self.throttle = throttle + self.use_get = use_get + if max_retries is None: + self.max_retries = pywikibot.config.max_retries else: - self.mime = kwargs.pop('mime', False) - self.throttle = kwargs.pop('throttle', True) - self.use_get = kwargs.pop('use_get', None) - self.max_retries = kwargs.pop("max_retries", pywikibot.config.max_retries) - self.retry_wait = kwargs.pop("retry_wait", pywikibot.config.retry_wait) + self.max_retries = max_retries + if retry_wait is None: + self.retry_wait = pywikibot.config.retry_wait + else: + self.retry_wait = retry_wait + # The only problem with that system is that it won't detect when + # 'parameters' is actually the only parameter for the request as it then + # assumes it's using the new mode (and the parameters are actually in + # the parameter 'parameters' not that the parameter 'parameters' is + # actually a parameter for the request). But that is invalid anyway as + # it MUST have at least an action parameter for the request which would + # be in kwargs if it's using the old mode. + if kwargs: + if parameters is not self._PARAM_DEFAULT: + # 'parameters' AND kwargs is set. In that case think of 'parameters' + # being an old kwarg which is now filled in an actual parameter + self._warn_both() + kwargs['parameters'] = parameters + # When parameters wasn't set it's likely that kwargs-mode was used + self._warn_kwargs() + parameters = kwargs + elif parameters is self._PARAM_DEFAULT: + parameters = {} self._params = {} - if "action" not in kwargs: + if "action" not in parameters: raise ValueError("'action' specification missing from Request.") - self.action = kwargs['action'] - self.update(**kwargs) + self.action = parameters['action'] + self.update(parameters) self._warning_handler = None # Actions that imply database updates on the server, used for various # things like throttling or skipping actions when we're in simulation @@ -1188,20 +1246,96 @@ self.site = EnableSSLSiteWrapper(self.site) @classmethod - def _format_value(cls, value): + def create_simple(cls, site, **kwargs): + """Create a new instance using all arguments except site for the API.""" + # This ONLY support site so that any caller can be sure there will be + # no conflict with PWB parameters + # TODO: Use ParamInfo request to determine valid parameters + if isinstance(kwargs.get('parameters'), dict): + warn('The request contains already a "parameters" entry which is a ' + 'dict.') + return cls(site, parameters=kwargs) + + @classmethod + def _warn_both(cls): + """Warn that kwargs mode was used but parameters was set too.""" + warn('Both kwargs and parameters are set in Request.__init__. It ' + 'assumes that "parameters" is actually a parameter of the ' + 'Request and is added to kwargs.', DeprecationWarning, 3) + + @classmethod + def _warn_kwargs(cls): + """Warn that kwargs was used instead of parameters.""" + warn('Instead of using kwargs from Request.__init__, parameters ' + 'for the request to the API should be added via the ' + '"parameters" parameter.', DeprecationWarning, 3) + + @classmethod + def clean_kwargs(cls, kwargs): + """ + Convert keyword arguments into new parameters mode. + + If there are no other arguments in kwargs apart from the used arguments + by the class' constructor it'll just return kwargs and otherwise remove + those which aren't in the constructor and put them in a dict which is + added as a 'parameters' keyword. It will always create a shallow copy. + + @param kwargs: The original keyword arguments which is not modified. + @type kwargs: dict + @return: The normalized keyword arguments. + @rtype: dict + """ + args = set() + for super_cls in inspect.getmro(cls): + if not super_cls.__name__.endswith('Request'): + break + args |= set(inspect.getargspec(super_cls.__init__)[0]) + else: + raise ValueError('Request was not a super class of ' + '{0!r}'.format(cls)) + args -= set(['self']) + old_kwargs = set(kwargs) + # all kwargs defined above but not in args indicate 'kwargs' mode + if old_kwargs - args: + # Move all kwargs into parameters + parameters = dict((name, value) for name, value in kwargs.items() + if name not in args or name == 'parameters') + if 'parameters' in parameters: + cls._warn_both() + # Copy only arguments and not the parameters + kwargs = dict((name, value) for name, value in kwargs.items() + if name in args or name == 'self') + kwargs['parameters'] = parameters + # Make sure that all arguments have remained + assert(old_kwargs | set(['parameters']) == + set(kwargs) | set(kwargs['parameters'])) + assert(('parameters' in old_kwargs) is + ('parameters' in kwargs['parameters'])) + cls._warn_kwargs() + else: + kwargs = dict(kwargs) + if 'parameters' not in kwargs: + kwargs['parameters'] = {} + return kwargs + + def _format_value(self, value): """ Format the MediaWiki API request parameter. Converts from Python datatypes to MediaWiki API parameter values. Supports: - * datetime.datetime + * datetime.datetime (using strftime and ISO8601 format) + * pywikibot.page.BasePage (using title (+namespace; -section)) All other datatypes are converted to string using unicode() on Python 2 and str() on Python 3. """ if isinstance(value, datetime.datetime): return value.strftime(pywikibot.Timestamp.ISO8601Format) + elif isinstance(value, pywikibot.page.BasePage): + assert(value.site == self.site) + return value.title(withSection=False) else: return unicode(value) @@ -1775,6 +1909,11 @@ self._cachetime = None @classmethod + def create_simple(cls, site, **kwargs): + """Unsupported as it requires at least two parameters.""" + raise NotImplementedError('CachedRequest cannot be created simply.') + + @classmethod def _get_cache_dir(cls): """Return the base directory path for cache entries. @@ -2028,26 +2167,24 @@ documentation for values. 'action'='query' is assumed. """ - if "action" in kwargs and kwargs["action"] != "query": + if not hasattr(self, 'site'): + kwargs = self._clean_kwargs(kwargs) # hasn't been called yet + parameters = kwargs['parameters'] + if 'action' in parameters and parameters['action'] != 'query': raise Error("%s: 'action' must be 'query', not %s" % (self.__class__.__name__, kwargs["action"])) else: - kwargs["action"] = "query" - try: - self.site = kwargs["site"] - except KeyError: - self.site = pywikibot.Site() - kwargs["site"] = self.site + parameters['action'] = 'query' # make sure request type is valid, and get limit key if any for modtype in ("generator", "list", "prop", "meta"): - if modtype in kwargs: - self.modules = kwargs[modtype].split('|') + if modtype in parameters: + self.modules = parameters[modtype].split('|') break else: raise Error("%s: No query module name found in arguments." % self.__class__.__name__) - kwargs['indexpageids'] = True # always ask for list of pageids + parameters['indexpageids'] = True # always ask for list of pageids if MediaWikiVersion(self.site.version()) < MediaWikiVersion('1.21'): self.continue_name = 'query-continue' self.continue_update = self._query_continue @@ -2055,8 +2192,8 @@ self.continue_name = 'continue' self.continue_update = self._continue # Explicitly enable the simplified continuation - kwargs['continue'] = True - self.request = Request(**kwargs) + parameters['continue'] = True + self.request = self.request_class(**kwargs) # This forces all paraminfo for all query modules to be bulk loaded. limited_modules = ( @@ -2098,12 +2235,12 @@ self.prefix = self.site._paraminfo['query+' + self.limited_module]['prefix'] self._update_limit() - if self.api_limit is not None and "generator" in kwargs: + if self.api_limit is not None and 'generator' in parameters: self.prefix = "g" + self.prefix self.limit = None self.query_limit = self.api_limit - if "generator" in kwargs: + if 'generator' in parameters: self.resultkey = "pages" # name of the "query" subelement key else: # to look for when iterating self.resultkey = self.modules[0] @@ -2117,6 +2254,18 @@ # "templates":{"tlcontinue":"310820|828|Namespace_detect"}} # self.continuekey is a list self.continuekey = self.modules + + def _clean_kwargs(self, kwargs, **mw_api_args): + """Clean kwargs, define site and request class.""" + if 'site' not in kwargs: + warn('QueryGenerator() invoked without a site', RuntimeWarning, 3) + kwargs['site'] = pywikibot.Site() + assert(not hasattr(self, 'site') or self.site == kwargs['site']) + self.site = kwargs['site'] + self.request_class = kwargs['site']._request_class(kwargs) + kwargs = self.request_class.clean_kwargs(kwargs) + kwargs['parameters'].update(mw_api_args) + return kwargs def set_query_increment(self, value): """Set the maximum number of items to be retrieved per API query. @@ -2386,16 +2535,19 @@ params[key] += '|' + value else: params[key] = value + kwargs = self._clean_kwargs(kwargs) + parameters = kwargs['parameters'] # get some basic information about every page generated - appendParams(kwargs, 'prop', 'info|imageinfo|categoryinfo') + appendParams(parameters, 'prop', 'info|imageinfo|categoryinfo') if g_content: # retrieve the current revision - appendParams(kwargs, 'prop', 'revisions') - appendParams(kwargs, 'rvprop', 'ids|timestamp|flags|comment|user|content') - if not ('inprop' in kwargs and 'protection' in kwargs['inprop']): - appendParams(kwargs, 'inprop', 'protection') - appendParams(kwargs, 'iiprop', 'timestamp|user|comment|url|size|sha1|metadata') - QueryGenerator.__init__(self, generator=generator, **kwargs) + appendParams(parameters, 'prop', 'revisions') + appendParams(parameters, 'rvprop', 'ids|timestamp|flags|comment|user|content') + if not ('inprop' in parameters and 'protection' in parameters['inprop']): + appendParams(parameters, 'inprop', 'protection') + appendParams(parameters, 'iiprop', 'timestamp|user|comment|url|size|sha1|metadata') + parameters['generator'] = generator + QueryGenerator.__init__(self, **kwargs) self.resultkey = "pages" # element to look for in result # TODO: Bug T91912 when using step > 50 with proofread, with queries @@ -2465,7 +2617,8 @@ @type prop: str """ - QueryGenerator.__init__(self, prop=prop, **kwargs) + kwargs = self._clean_kwargs(kwargs, prop=prop) + QueryGenerator.__init__(self, **kwargs) self._props = frozenset(prop.split('|')) self.resultkey = "pages" @@ -2502,7 +2655,8 @@ @type listaction: str """ - QueryGenerator.__init__(self, list=listaction, **kwargs) + kwargs = self._clean_kwargs(kwargs, list=listaction) + QueryGenerator.__init__(self, **kwargs) class LogEntryListGenerator(ListGenerator): @@ -2543,11 +2697,11 @@ pywikibot.warning(u"Too many tries, waiting %s seconds before retrying." % diff.seconds) time.sleep(diff.seconds) - login_request = Request(site=self.site, - use_get=False, - action="login", - lgname=self.username, - lgpassword=self.password) + login_request = self.site._request( + use_get=False, + parameters=dict(action='login', + lgname=self.username, + lgpassword=self.password)) self.site._loginstatus = -2 while True: login_result = login_request.submit() diff --git a/pywikibot/page.py b/pywikibot/page.py index 5bbb043..7c27551 100644 --- a/pywikibot/page.py +++ b/pywikibot/page.py @@ -2856,7 +2856,7 @@ } if ccme: params['ccme'] = 1 - mailrequest = pywikibot.data.api.Request(site=self.site, **params) + mailrequest = self.site._simple_request(**params) maildata = mailrequest.submit() if 'emailuser' in maildata: diff --git a/pywikibot/site.py b/pywikibot/site.py index 130ddb8..d2932ce 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -36,6 +36,7 @@ deprecated, deprecate_arg, deprecated_args, remove_last_args, redirect_func, issue_deprecation_warning, manage_wrapping, MediaWikiVersion, first_upper, normalize_username, + merge_unique_dicts, ) from pywikibot.tools.ip import is_IP from pywikibot.throttle import Throttle @@ -1234,12 +1235,14 @@ raise ValueError('At least one property name must be provided.') invalid_properties = [] try: - request = pywikibot.data.api.CachedRequest( + request = self._site._request( expiry=pywikibot.config.API_config_expiry if expiry is False else expiry, - site=self._site, - action='query', - meta='siteinfo', - siprop=props) + parameters=dict( + action='query', + meta='siteinfo', + siprop=props + ) + ) # With 1.25wmf5 it'll require continue or rawcontinue. As we don't # continue anyway we just always use continue. request['continue'] = True @@ -1582,9 +1585,9 @@ @classmethod def fromDBName(cls, dbname): # TODO this only works for some WMF sites - req = api.CachedRequest(datetime.timedelta(days=10), - site=pywikibot.Site('meta', 'meta'), - action='sitematrix') + site = pywikibot.Site('meta', 'meta') + req = site._request(expiry=datetime.timedelta(days=10), + parameters={'action': 'sitematrix'}) data = req.submit() for key, val in data['sitematrix'].items(): if key == 'count': @@ -1606,8 +1609,9 @@ step=None, total=None, **args): """Convenience method that returns an API generator. - All keyword args not listed below are passed to the generator's - constructor unchanged. + All generic keyword arguments are passed as MW API parameter except for + 'g_content' which is passed as a normal parameter to the generator's + constructor. @param gen_class: the type of generator to construct (must be a subclass of pywikibot.data.api.QueryGenerator) @@ -1630,10 +1634,14 @@ @raises TypeError: a namespace identifier has an inappropriate type such as NoneType or bool """ + # TODO: Support parameters/simple modes? + req_args = {'site': self, 'parameters': args} + if 'g_content' in args: + req_args['g_content'] = args.pop('g_content') if type_arg is not None: - gen = gen_class(type_arg, site=self, **args) + gen = gen_class(type_arg, **req_args) else: - gen = gen_class(site=self, **args) + gen = gen_class(**req_args) if namespaces is not None: gen.set_namespace(namespaces) if step is not None and int(step) > 0: @@ -1641,6 +1649,30 @@ if total is not None and int(total) > 0: gen.set_maximum_items(int(total)) return gen + + def _request_class(self, kwargs): + """ + Get the appropriate class. + + Inside this class kwargs use the parameters mode but QueryGenerator may + use the old kwargs mode. + """ + # This checks expiry in kwargs and not kwargs['parameters'] so it won't + # create a CachedRequest when there is an expiry in an API parameter + # and kwargs here are actually in parameters mode. + if 'expiry' in kwargs: + return api.CachedRequest + else: + return api.Request + + def _request(self, **kwargs): + """Create a request by forwarding all parameters directly.""" + return self._request_class(kwargs)(site=self, **kwargs) + + def _simple_request(self, **kwargs): + """Create a request by defining all kwargs as parameters.""" + return self._request_class({'parameters': kwargs}).create_simple( + site=self, **kwargs) def logged_in(self, sysop=False): """Verify the bot is logged into the site as the expected user. @@ -1736,7 +1768,7 @@ Also logs out of the global account if linked to the user. """ - uirequest = api.Request(site=self, action="logout") + uirequest = self._simple_request(action='logout') uirequest.submit() self._loginstatus = LoginStatus.NOT_LOGGED_IN if hasattr(self, "_userinfo"): @@ -1760,8 +1792,7 @@ if (not hasattr(self, '_userinfo') or 'rights' not in self._userinfo or self._userinfo['name'] != self._username['sysop' in self._userinfo['groups']]): - uirequest = api.Request( - site=self, + uirequest = self._simple_request( action="query", meta="userinfo", uiprop="blockinfo|hasmsg|groups|rights" @@ -1790,8 +1821,7 @@ """ if not hasattr(self, "_globaluserinfo"): - uirequest = api.Request( - site=self, + uirequest = self._simple_request( action="query", meta="globaluserinfo", guiprop="groups|rights|editcount" @@ -1851,8 +1881,7 @@ # TODO: Integrate into _userinfo if (force or not hasattr(self, '_useroptions') or self.user() != self._useroptions['_name']): - uirequest = api.Request( - site=self, + uirequest = self._simple_request( action="query", meta="userinfo", uiprop="options" @@ -1915,14 +1944,14 @@ @need_extension('Echo') def notifications(self, **kwargs): """Yield Notification objects from the Echo extension.""" - params = dict(site=self, action='query', + params = dict(action='query', meta='notifications', notprop='list', notformat='text') for key in kwargs: params['not' + key] = kwargs[key] - data = api.Request(**params).submit() + data = self._simple_request(**params).submit() for notif in data['query']['notifications']['list'].values(): yield Notification.fromJSON(self, notif) @@ -1935,10 +1964,9 @@ """ # TODO: ensure that the 'echomarkread' action # is supported by the site - req = api.Request(site=self, - action='echomarkread', - token=self.tokens['edit'], - **kwargs) + kwargs = merge_unique_dicts(kwargs, action='echomarkread', + token=self.tokens['edit']) + req = self._simple_request(**kwargs) data = req.submit() try: return data['query']['echomarkread']['result'] == 'success' @@ -2095,7 +2123,7 @@ raise ValueError('text must be a string') if not text: return '' - req = api.Request(site=self, action='expandtemplates', text=text) + req = self._simple_request(action='expandtemplates', text=text) if title is not None: req['title'] = title if includecomments is True: @@ -2374,12 +2402,13 @@ not hasattr(self, '_proofread_page_ns') or not hasattr(self, '_proofread_levels')): - pirequest = api.CachedRequest( - site=self, + pirequest = self._request( expiry=pywikibot.config.API_config_expiry if expiry is False else expiry, - action='query', - meta='proofreadinfo', - piprop='namespaces|qualitylevels' + parameters=dict( + action='query', + meta='proofreadinfo', + piprop='namespaces|qualitylevels' + ) ) pidata = pirequest.submit() @@ -2601,10 +2630,12 @@ return page._redirtarget title = page.title(withSection=False) - query = api.Request(site=self, action="query", prop="info", - inprop="protection|talkid|subjectid", - titles=title.encode(self.encoding()), - redirects="") + query = self._simple_request( + action='query', + prop='info', + inprop=['protection', 'talkid', 'subjectid'], + titles=title, + redirects=True) result = query.submit() if "query" not in result or "redirects" not in result["query"]: raise RuntimeError( @@ -2835,9 +2866,9 @@ if MediaWikiVersion('1.14') <= _version < MediaWikiVersion('1.17'): user_tokens['patrol'] = user_tokens['edit'] else: - req = api.Request(site=self, action='query', - list='recentchanges', - rctoken='patrol', rclimit=1) + req = self._simple_request(action='query', + list='recentchanges', + rctoken='patrol', rclimit=1) req._warning_handler = warn_handler data = req.submit() @@ -2855,16 +2886,16 @@ types_wiki = self._paraminfo.parameter('tokens', 'type')['type'] types.extend(types_wiki) - req = api.Request(site=self, action='tokens', - type='|'.join(self.validate_tokens(types))) + req = self._simple_request(action='tokens', + type=self.validate_tokens(types)) else: if all is not False: types_wiki = self._paraminfo.parameter('query+tokens', 'type')['type'] types.extend(types_wiki) - req = api.Request(site=self, action='query', meta='tokens', - type='|'.join(self.validate_tokens(types))) + req = self._simple_request(action='query', meta='tokens', + type=self.validate_tokens(types)) req._warning_handler = warn_handler data = req.submit() @@ -4242,8 +4273,7 @@ if bot is None: bot = ("bot" in self.userinfo["rights"]) self.lock_page(page) - params = dict(action="edit", - title=page.title(withSection=False), + params = dict(action='edit', title=page, text=text, token=token, summary=summary, bot=bot, recreate=recreate, createonly=createonly, nocreate=nocreate, minor=minor, @@ -4266,7 +4296,7 @@ pywikibot.warning( u"editpage: Invalid watch value '%(watch)s' ignored." % locals()) - req = api.Request(site=self, **params) + req = self._simple_request(**params) while True: try: result = req.submit() @@ -4410,9 +4440,12 @@ "does not exist on %(site)s.") token = self.tokens['move'] self.lock_page(page) - req = api.Request(site=self, action="move", to=newtitle, - token=token, reason=summary, movetalk=movetalk, - noredirect=noredirect) + req = self._simple_request(action='move', + noredirect=noredirect, + reason=summary, + movetalk=movetalk, + token=token, + to=newtitle) req['from'] = oldtitle # "from" is a python keyword try: result = req.submit() @@ -4501,13 +4534,12 @@ raise Error( u"Rollback of %s aborted; only one user in revision history." % page.title(asLink=True)) - token = self.tokens["rollback"] + parameters = merge_unique_dicts(kwargs, action='rollback', + title=page, + token=self.tokens['rollback'], + user=last_user) self.lock_page(page) - req = api.Request(site=self, action="rollback", - title=page.title(withSection=False), - user=last_user, - token=token, - **kwargs) + req = self._simple_request(**parameters) try: req.submit() except api.APIError as err: @@ -4549,9 +4581,10 @@ """ token = self.tokens['delete'] self.lock_page(page) - req = api.Request(site=self, action="delete", token=token, - title=page.title(withSection=False), - reason=reason) + req = self._simple_request(action='delete', + token=token, + title=page, + reason=reason) try: req.submit() except api.APIError as err: @@ -4585,13 +4618,11 @@ token = self.tokens['undelete'] self.lock_page(page) - if revisions is None: - req = api.Request(site=self, action='undelete', token=token, - title=page.title(withSection=False), reason=reason) - else: - req = api.Request(site=self, action='undelete', token=token, - title=page.title(withSection=False), - timestamps=revisions, reason=reason) + req = self._simple_request(action='undelete', + title=page, + reason=reason, + token=token, + timestamps=revisions) try: req.submit() except api.APIError as err: @@ -4662,13 +4693,12 @@ protectList = [ptype + '=' + level for ptype, level in protections.items() if level is not None] - req = api.Request(site=self, action='protect', token=token, - title=page.title(withSection=False), - protections=protectList, - reason=reason, - **kwargs) - if expiry: - req['expiry'] = expiry + parameters = merge_unique_dicts(kwargs, action='protect', title=page, + token=token, + protections=protectList, reason=reason, + expiry=expiry) + + req = self._simple_request(**parameters) try: req.submit() except api.APIError as err: @@ -4754,8 +4784,9 @@ token = self.tokens['patrol'] for idvalue, idtype in gen: - req = api.Request(site=self, action='patrol', - token=token, **{idtype: idvalue}) + req = self._request(parameters={'action': 'patrol', + 'token': token, + idtype: idvalue}) try: result = req.submit() @@ -4817,11 +4848,11 @@ token = self.tokens['block'] if expiry is False: expiry = 'never' - req = api.Request(site=self, action='block', user=user.username, - expiry=expiry, reason=reason, token=token, - anononly=anononly, nocreate=nocreate, - autoblock=autoblock, noemail=noemail, - reblock=reblock) + req = self._simple_request(action='block', user=user.username, + expiry=expiry, reason=reason, token=token, + anononly=anononly, nocreate=nocreate, + autoblock=autoblock, noemail=noemail, + reblock=reblock) data = req.submit() return data @@ -4836,9 +4867,10 @@ @param reason: Reason for the unblock. @type reason: basestring """ - token = self.tokens['block'] - req = api.Request(site=self, action='unblock', user=user.username, - reason=reason, token=token) + req = self._simple_request(action='unblock', + user=user.username, + token=self.tokens['block'], + reason=reason) data = req.submit() return data @@ -4852,9 +4884,13 @@ @return: True if API returned expected response; False otherwise """ - token = self.tokens['watch'] - req = api.Request(site=self, action='watch', token=token, - title=page.title(withSection=False), unwatch=unwatch) + # TODO: Separated parameters to allow easy conversion to 'watchpages' + # as the API now allows 'titles'. + parameters = {'action': 'watch', + 'title': page, + 'token': self.tokens['watch'], + 'unwatch': unwatch} + req = self._simple_request(**parameters) result = req.submit() if "watch" not in result: pywikibot.error(u"watchpage: Unexpected API response:\n%s" % result) @@ -4869,8 +4905,8 @@ @return: True if API returned expected response; False otherwise """ - req = api.Request(site=self, action='purge') - req['titles'] = [page.title(withSection=False) for page in set(pages)] + req = self._simple_request(action='purge', + titles=[page for page in set(pages)]) linkupdate = False linkupdate_args = ['forcelinkupdate', 'forcerecursivelinkupdate'] for arg in kwargs: @@ -4939,9 +4975,9 @@ # filekey, file, url, statuskey is required # TODO: is there another way? try: - token = self.tokens['edit'] - req = api.Request(site=self, action="upload", - token=token, throttle=False) + req = self._request(throttle=False, + parameters={'action': 'upload', + 'token': self.tokens['edit']}) req.submit() except api.APIError as error: if error.code == u'uploaddisabled': @@ -5030,10 +5066,10 @@ pywikibot.log('Reused already upload file using ' 'filekey "{0}"'.format(_file_key)) # TODO: Use sessionkey instead of filekey if necessary - req = api.Request(site=self, action='upload', token=token, - filename=file_page_title, - comment=comment, text=text, - filekey=_file_key) + final_request = self._simple_request(action='upload', token=token, + filename=file_page_title, + comment=comment, text=text, + filekey=_file_key) elif source_filename: # TODO: Dummy value to allow also Unicode names, see bug 73661 mime_filename = 'FAKE-NAME' @@ -5042,12 +5078,15 @@ if not os.path.isfile(source_filename): raise ValueError("File '%s' does not exist." % source_filename) - additional_parameters = {} throttle = True filesize = os.path.getsize(source_filename) chunked_upload = (chunk_size > 0 and chunk_size < filesize and MediaWikiVersion(self.version()) >= MediaWikiVersion('1.20')) with open(source_filename, 'rb') as f: + final_request = self._request( + throttle=throttle, parameters={ + 'action': 'upload', 'token': token, 'text': text, + 'filename': file_page_title, 'comment': comment}) if chunked_upload: offset = _offset if offset > 0: @@ -5056,11 +5095,16 @@ while True: f.seek(offset) chunk = f.read(chunk_size) - req = api.Request(site=self, action='upload', token=token, - stash=True, offset=offset, filesize=filesize, - filename=file_page_title, - ignorewarnings=ignore_warnings, - mime_params={}, throttle=throttle) + req = self._request( + throttle=throttle, mime=True, + parameters={ + 'action': 'upload', + 'token': token, + 'stash': True, + 'filesize': filesize, + 'offset': offset, + 'filename': file_page_title, + 'ignorewarnings': ignore_warnings}) req.mime_params['chunk'] = (chunk, ("application", "octet-stream"), {'filename': mime_filename}) @@ -5093,37 +5137,30 @@ offset += len(chunk) if data['result'] != 'Continue': # finished pywikibot.log('Finished uploading last chunk.') - additional_parameters['filekey'] = _file_key + final_request['filekey'] = _file_key break else: # not chunked upload file_contents = f.read() filetype = (mimetypes.guess_type(source_filename)[0] or 'application/octet-stream') - additional_parameters = { - 'mime_params': { - 'file': (file_contents, - filetype.split('/'), - {'filename': mime_filename}) - } + final_request.mime_params = { + 'file': (file_contents, filetype.split('/'), + {'filename': mime_filename}) } - req = api.Request(site=self, action="upload", token=token, - filename=file_page_title, - comment=comment, text=text, throttle=throttle, - **additional_parameters) else: # upload by URL if "upload_by_url" not in self.userinfo["rights"]: raise Error( "User '%s' is not authorized to upload by URL on site %s." % (self.user(), self)) - req = api.Request(site=self, action="upload", token=token, - filename=file_page_title, - url=source_url, comment=comment, text=text) + final_request = self._simple_request( + action='upload', filename=file_page_title, + url=source_url, comment=comment, text=text, token=token) if not result: - req['watch'] = watch - req['ignorewarnings'] = ignore_warnings + final_request['watch'] = watch + final_request['ignorewarnings'] = ignore_warnings try: - result = req.submit() + result = final_request.submit() self._uploaddisabled = False except api.APIError as error: # TODO: catch and process foreseeable errors @@ -5553,10 +5590,11 @@ if not diff: raise TypeError('diff parameter is of invalid type') - params = {'from{0}'.format(old[0]): old[1], + params = {'action': 'compare', + 'from{0}'.format(old[0]): old[1], 'to{0}'.format(diff[0]): diff[1]} - req = api.Request(site=self, action='compare', **params) + req = self._simple_request(**params) data = req.submit() comparison = data['compare']['*'] return comparison @@ -5571,9 +5609,8 @@ @return: A dict representing the board's data. @rtype: dict """ - title = page.title(withSection=False) - req = api.Request(site=self, action='flow', page=title, - submodule='view-topiclist') + req = self._simple_request(action='flow', page=page, + submodule='view-topiclist') data = req.submit() return data['flow']['view-topiclist']['result']['topiclist'] @@ -5586,9 +5623,8 @@ @return: A dict representing the topic's data. @rtype: dict """ - title = page.title(withSection=False) - req = api.Request(site=self, action='flow', page=title, - submodule='view-topic') + req = self._simple_request(action='flow', page=page, + submodule='view-topic') data = req.submit() return data['flow']['view-topic']['result']['topic'] @@ -5699,8 +5735,8 @@ if isinstance(source, int) or \ isinstance(source, basestring) and source.isdigit(): ids = 'q' + str(source) - wbrequest = api.Request(site=self, action="wbgetentities", ids=ids, - **params) + params = merge_unique_dicts(params, action='wbgetentities', ids=ids) + wbrequest = self._simple_request(**params) wbdata = wbrequest.submit() assert 'success' in wbdata, \ "API wbgetentities response lacks 'success' key" @@ -5727,11 +5763,12 @@ @type identification: dict @param props: the optional properties to fetch. """ - params = dict(**identification) - params['action'] = 'wbgetentities' - if props: - params['props'] = '|'.join(props) - req = api.Request(site=self, **params) + params = merge_unique_dicts(identification, action='wbgetentities', + # TODO: When props is empty it results in + # an empty string ('&props=') but it should + # result in a missing entry. + props=props if props else False) + req = self._simple_request(**params) data = req.submit() if 'success' not in data: raise api.APIError(data['errors']) @@ -5765,7 +5802,7 @@ req['sites'].append(p.site.dbName()) req['titles'].append(p._link._text) - req = api.Request(site=self, action='wbgetentities', **req) + req = self._simple_request(action='wbgetentities', **req) data = req.submit() for qid in data['entities']: item = pywikibot.ItemPage(self, qid) @@ -5786,7 +5823,7 @@ ) expiry = datetime.timedelta(days=365 * 100) # Store it for 100 years - req = api.CachedRequest(expiry, site=self, **params) + req = self._request(expiry=expiry, parameters=params) data = req.submit() # the IDs returned from the API can be upper or lowercase, depending @@ -5817,7 +5854,7 @@ if arg in ['clear', 'data', 'exclude', 'summary']: params[arg] = kwargs[arg] params['data'] = json.dumps(data) - req = api.Request(site=self, **params) + req = self._simple_request(**params) data = req.submit() return data @@ -5837,7 +5874,7 @@ if 'summary' in kwargs: params['summary'] = kwargs['summary'] params['token'] = self.tokens['edit'] - req = api.Request(site=self, **params) + req = self._simple_request(**params) data = req.submit() claim.snak = data['claim']['id'] # Update the item @@ -5875,7 +5912,7 @@ params['value'] = json.dumps(claim._formatValue()) params['baserevid'] = claim.on_item.lastrevid - req = api.Request(site=self, **params) + req = self._simple_request(**params) data = req.submit() return data @@ -5902,7 +5939,7 @@ if 'summary' in kwargs: params['summary'] = kwargs['summary'] - req = api.Request(site=self, **params) + req = self._simple_request(**params) data = req.submit() return data @@ -5956,7 +5993,7 @@ if arg in ['baserevid', 'summary']: params[arg] = kwargs[arg] - req = api.Request(site=self, **params) + req = self._simple_request(**params) data = req.submit() return data @@ -5994,7 +6031,7 @@ if arg in ['baserevid', 'summary']: params[arg] = kwargs[arg] - req = api.Request(site=self, **params) + req = self._simple_request(**params) data = req.submit() return data @@ -6008,7 +6045,7 @@ for kwarg in kwargs: if kwarg in ['baserevid', 'summary']: params[kwarg] = kwargs[kwarg] - req = api.Request(site=self, **params) + req = self._simple_request(**params) data = req.submit() return data @@ -6031,7 +6068,7 @@ for kwarg in kwargs: if kwarg in ['baserevid', 'summary']: params[kwarg] = kwargs[kwarg] - req = api.Request(site=self, **params) + req = self._simple_request(**params) data = req.submit() return data @@ -6056,7 +6093,7 @@ } if bot: params['bot'] = 1 - req = api.Request(site=self, **params) + req = self._simple_request(**params) data = req.submit() return data @@ -6079,7 +6116,7 @@ for kwarg in kwargs: if kwarg in ['ignoreconflicts', 'summary']: params[kwarg] = kwargs[kwarg] - req = api.Request(site=self, **params) + req = self._simple_request(**params) data = req.submit() return data @@ -6098,7 +6135,7 @@ 'to': to_item.getID(), 'token': self.tokens['edit'] } - req = api.Request(site=self, **params) + req = self._simple_request(**params) data = req.submit() return data diff --git a/pywikibot/tools/__init__.py b/pywikibot/tools/__init__.py index 8d9c4ff..3a836db 100644 --- a/pywikibot/tools/__init__.py +++ b/pywikibot/tools/__init__.py @@ -25,6 +25,7 @@ if sys.version_info[0] > 2: import queue as Queue basestring = (str,) + unicode = str else: import Queue @@ -816,6 +817,25 @@ return open(filename, 'rb') +def merge_unique_dicts(*args, **kwargs): + """ + Return a merged dict and making sure that the original dicts had unique keys. + + The positional arguments are the dictionaries to be merged. It is also + possible to define an additional dict using the keyword arguments. + """ + args = list(args) + [dict(kwargs)] + conflicts = set() + result = {} + for arg in args: + conflicts |= set(arg.keys()) & set(result.keys()) + result.update(arg) + if conflicts: + raise ValueError('Multiple dicts contain the same keys: ' + '{0}'.format(', '.join(sorted(unicode(key) for key in conflicts)))) + return result + + # Decorators # # Decorator functions without parameters are _invoked_ differently from diff --git a/scripts/casechecker.py b/scripts/casechecker.py index e7aff0a..33afb18 100755 --- a/scripts/casechecker.py +++ b/scripts/casechecker.py @@ -267,7 +267,7 @@ 'pllimit': 'max', } - req = api.Request(site=self.site, **wlparams) + req = api.Request(site=self.site, parameters=wlparams) data = req.submit() if len(data['query']['pageids']) == 1: pageid = data['query']['pageids'][0] @@ -728,7 +728,7 @@ 'bllimit': '50', } - req = api.Request(**params) + req = api.Request(site=self.site, parameters=params) data = req.submit() cl = 0 redirs = 0 diff --git a/scripts/redirect.py b/scripts/redirect.py index c8ba68b..5b4f4eb 100755 --- a/scripts/redirect.py +++ b/scripts/redirect.py @@ -229,8 +229,10 @@ where chain or loop detecton was halted, or None if unknown """ for apiQ in self._next_redirect_group(): - gen = pywikibot.data.api.Request(site=self.site, action='query', - redirects=True, pageids=apiQ) + gen = pywikibot.data.api.Request( + site=self.site, parameters={'action': 'query', + 'redirects': True, + 'pageids': apiQ}) data = gen.submit() if 'error' in data: raise RuntimeError("API query error: %s" % data) diff --git a/scripts/revertbot.py b/scripts/revertbot.py index abd3fbf..c905f1f 100755 --- a/scripts/revertbot.py +++ b/scripts/revertbot.py @@ -111,8 +111,10 @@ page.save(comment) return comment try: - pywikibot.data.api.Request(action="rollback", title=page.title(), user=self.user, - token=rev[4], markbot=1).submit() + pywikibot.data.api.Request( + self.site, parameters={'action': 'rollback', 'title': page, + 'user': self.user, 'token': rev[4], + 'markbot': True}).submit() except pywikibot.data.api.APIError as e: if e.code == 'badtoken': pywikibot.error("There was an API token error rollbacking the edit") diff --git a/tests/__init__.py b/tests/__init__.py index 429e5c8..9f43a5b 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -209,6 +209,12 @@ """Constructor.""" super(TestRequest, self).__init__(0, *args, **kwargs) + @classmethod + def create_simple(cls, **kwargs): + """Circumvent CachedRequest implementation.""" + site = kwargs.pop('site') + return cls(site=site, parameters=kwargs) + def _expired(self, dt): """Never invalidate cached data.""" return False diff --git a/tests/api_tests.py b/tests/api_tests.py index dde46f5..5cc9aa9 100644 --- a/tests/api_tests.py +++ b/tests/api_tests.py @@ -61,6 +61,16 @@ for item in req.items(): self.assertEqual(len(item), 2, item) + def test_mixed_mode(self): + """Test if parameters is used with kwargs.""" + req1 = api.Request(site=self.site, action='test', parameters='foo') + self.assertIn('parameters', req1._params) + + req2 = api.Request(site=self.site, parameters={'action': 'test', + 'parameters': 'foo'}) + self.assertEqual(req2['parameters'], ['foo']) + self.assertEqual(req1._params, req2._params) + class TestParamInfo(DefaultSiteTestCase): diff --git a/tests/tools_tests.py b/tests/tools_tests.py index 718f589..434cab5 100644 --- a/tests/tools_tests.py +++ b/tests/tools_tests.py @@ -116,6 +116,43 @@ self.assertRaises(OSError, self._get_content, self.base_file + '_invalid.7z', True) +class MergeUniqueDicts(TestCase): + + """Test merge_unique_dicts.""" + + net = False + dct1 = {'foo': 'bar', '42': 'answer'} + dct2 = {47: 'Star', 74: 'Trek'} + dct_both = dct1.copy() + dct_both.update(dct2) + + def test_single(self): + """Test that it returns the dict itself when there is only one.""" + self.assertEqual(tools.merge_unique_dicts(self.dct1), self.dct1) + self.assertEqual(tools.merge_unique_dicts(**self.dct1), self.dct1) + + def test_multiple(self): + """Test that it actually merges dicts.""" + self.assertEqual(tools.merge_unique_dicts(self.dct1, self.dct2), + self.dct_both) + self.assertEqual(tools.merge_unique_dicts(self.dct2, **self.dct1), + self.dct_both) + + def test_different_type(self): + """Test that the keys can be different types.""" + self.assertEqual(tools.merge_unique_dicts({'1': 'str'}, {1: 'int'}), + {'1': 'str', 1: 'int'}) + + def test_conflict(self): + """Test that it detects conflicts.""" + self.assertRaisesRegex( + ValueError, '42', tools.merge_unique_dicts, self.dct1, **{'42': 'bad'}) + self.assertRaisesRegex( + ValueError, '42', tools.merge_unique_dicts, self.dct1, self.dct1) + self.assertRaisesRegex( + ValueError, '42', tools.merge_unique_dicts, self.dct1, **self.dct1) + + if __name__ == '__main__': try: unittest.main() diff --git a/tests/utils.py b/tests/utils.py index dabb03c..ac5fc50 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -190,6 +190,11 @@ """Constructor.""" _original_Request.__init__(self, *args, **kwargs) + @classmethod + def create_simple(cls, **kwargs): + """Skip CachedRequest implementation.""" + return _original_Request.create_simple(**kwargs) + def _expired(self, dt): """Never invalidate cached data.""" return False -- To view, visit https://gerrit.wikimedia.org/r/216104 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ic259420c312d4efe4f14c772b369f86e6ed2e819 Gerrit-PatchSet: 15 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <commodorefabia...@gmx.de> Gerrit-Reviewer: John Vandenberg <jay...@gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgr...@gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhall...@arctus.nl> Gerrit-Reviewer: Ricordisamoa <ricordisa...@openmailbox.org> Gerrit-Reviewer: XZise <commodorefabia...@gmx.de> Gerrit-Reviewer: Xqt <i...@gno.de> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ Pywikibot-commits mailing list Pywikibot-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/pywikibot-commits