Xqt has uploaded a new change for review.
https://gerrit.wikimedia.org/r/288919
Change subject: [IMPR] Disable PreloadingGenerator when -liverecentchanges is
selected
......................................................................
[IMPR] Disable PreloadingGenerator when -liverecentchanges is selected
It is exceedingly superfluous to make a bulk load for -liverecentchanges which
waits for an event. Otherwise the process waits until the buffer is full to
start the operation. This is counterproductive because that option intends
processing the event as soon as possible.
Bug: T135331
Change-Id: I52777a8b2405686ad7a3dffdf631789752afb7ff
---
M pywikibot/pagegenerators.py
1 file changed, 41 insertions(+), 16 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core
refs/changes/19/288919/1
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index 6c7a2b2..4d81a23 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -347,7 +347,7 @@
__doc__ = __doc__.replace("¶ms;", parameterHelp)
-class GeneratorFactory(object):
+class GeneratorFactoryClass(object):
"""Process command line arguments and return appropriate page generator.
@@ -387,6 +387,7 @@
self._site = site
self._positional_arg_name = positional_arg_name
self._sparql = None
+ self._liverc = None
@property
def site(self):
@@ -714,6 +715,7 @@
_filter_unique=self._filter_unique)
elif arg == '-liverecentchanges':
+ self._liverc = True
gen = LiveRCPageGenerator(self.site, total=intNone(value))
elif arg == '-file':
if not value:
@@ -945,6 +947,24 @@
else:
return False
+
+_generator_factory = None
+
+def GeneratorFactory(*args, **kwargs):
+ """
+ A factory method to obtain a GeneratorFactoryClass object.
+
+ We assume there is only one GeneratorFactoryClass object inside a script.
+
+ @param site: Site for generator results.
+ @type site: L{pywikibot.site.BaseSite}
+ @param positional_arg_name: generator to use for positional args,
+ which do not begin with a hyphen
+ @type positional_arg_name: basestring
+ """
+ global _generator_factory
+ _generator_factory = GeneratorFactoryClass(*args, **kwargs)
+ return _generator_factory
@deprecated_args(step=None)
def AllpagesPageGenerator(start='!', namespace=0, includeredirects=True,
@@ -1857,22 +1877,27 @@
@param groupsize: how many pages to preload at once
@type groupsize: int
"""
- # pages may be on more than one site, for example if an interwiki
- # generator is used, so use a separate preloader for each site
- sites = {}
- # build a list of pages for each site found in the iterator
- for page in generator:
- site = page.site
- sites.setdefault(site, []).append(page)
- if len(sites[site]) >= groupsize:
- # if this site is at the groupsize, process it
- group = sites.pop(site)
- for i in site.preloadpages(group, groupsize):
+ if _generator_factory and _generator_factory._liverc:
+ # do not preload pages for -liverecentchanges option
+ for page in generator:
+ yield page
+ else:
+ # pages may be on more than one site, for example if an interwiki
+ # generator is used, so use a separate preloader for each site
+ sites = {}
+ # build a list of pages for each site found in the iterator
+ for page in generator:
+ site = page.site
+ sites.setdefault(site, []).append(page)
+ if len(sites[site]) >= groupsize:
+ # if this site is at the groupsize, process it
+ group = sites.pop(site)
+ for i in site.preloadpages(group, groupsize):
+ yield i
+ for site, pages in sites.items():
+ # process any leftover sites that never reached the groupsize
+ for i in site.preloadpages(pages, groupsize):
yield i
- for site, pages in sites.items():
- # process any leftover sites that never reached the groupsize
- for i in site.preloadpages(pages, groupsize):
- yield i
@deprecated_args(step='groupsize')
--
To view, visit https://gerrit.wikimedia.org/r/288919
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I52777a8b2405686ad7a3dffdf631789752afb7ff
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Xqt <i...@gno.de>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
