jenkins-bot has submitted this change and it was merged.
Change subject: Increase limits in QueryGenerator when data are sparse
......................................................................
Increase limits in QueryGenerator when data are sparse
In QueryGenerator's query loop, instead of just narrowing down limit
given to each api request.submit(), increase limit when only
"query-continue" data are fetched by a request.submit().
This will reduce the number of needed queries if data are "sparse".
See bug details for an example with data showing the background of the change.
Bug: 72209
Change-Id: I800a3399a991f5c51d23ed38b8636f09b704a9c6
---
M pywikibot/data/api.py
1 file changed, 31 insertions(+), 5 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
XZise: Looks good to me, but someone else must approve
jenkins-bot: Verified
diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index 5357cc5..9603381 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -565,10 +565,6 @@
rawdata = http.request(
self.site, uri, method="POST",
headers=headers, body=body)
-
-# import traceback
-# traceback.print_stack()
-# print rawdata
except Server504Error:
pywikibot.log(u"Caught HTTP 504 error; retrying")
self.wait()
@@ -1005,13 +1001,24 @@
Continues response as needed until limit (if any) is reached.
"""
+ previous_result_had_data = True
+ prev_limit = new_limit = None
+
count = 0
while True:
if self.query_limit is not None:
+ prev_limit = new_limit
if self.limit is None:
new_limit = self.query_limit
elif self.limit > 0:
- new_limit = min(self.query_limit, self.limit - count)
+ if previous_result_had_data:
+ # self.resultkey in data in last request.submit()
+ new_limit = min(self.query_limit, self.limit - count)
+ else:
+ # only "query-continue" returned. See Bug 72209.
+ # increase new_limit to advance faster until new
+ # useful data are found again.
+ new_limit = min(new_limit * 2, self.query_limit)
else:
new_limit = None
@@ -1025,6 +1032,20 @@
new_limit = min(new_limit, self.api_limit // 10, 250)
if new_limit is not None:
self.request[self.prefix + "limit"] = str(new_limit)
+ if prev_limit != new_limit:
+ pywikibot.debug(
+ u"%s: query_limit: %s, api_limit: %s, "
+ u"limit: %s, new_limit: %s, count: %s"
+ % (self.__class__.__name__,
+ self.query_limit, self.api_limit,
+ self.limit, new_limit, count),
+ _logger)
+ pywikibot.debug(
+ u"%s: %s: %s"
+ % (self.__class__.__name__,
+ self.prefix + "limit",
+ self.request[self.prefix + "limit"]),
+ _logger)
if not hasattr(self, "data"):
self.data = self.request.submit()
if not self.data or not isinstance(self.data, dict):
@@ -1085,12 +1106,17 @@
# note: self.limit could be -1
if self.limit and self.limit > 0 and count >= self.limit:
return
+ # self.resultkey in data in last request.submit()
+ previous_result_had_data = True
else:
# if query-continue is present, self.resultkey might not have
been
# fetched yet
if "query-continue" not in self.data:
# No results.
return
+ # self.resultkey not in data in last request.submit()
+ # only "query-continue" was retrieved.
+ previous_result_had_data = False
if self.module == "random" and self.limit:
# "random" module does not return "query-continue"
# now we loop for a new random query
--
To view, visit https://gerrit.wikimedia.org/r/167438
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I800a3399a991f5c51d23ed38b8636f09b704a9c6
Gerrit-PatchSet: 3
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Mpaa <mpaa.w...@gmail.com>
Gerrit-Reviewer: John Vandenberg <jay...@gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgr...@gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhall...@arctus.nl>
Gerrit-Reviewer: XZise <commodorefabia...@gmx.de>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
