John Vandenberg has uploaded a new change for review.
https://gerrit.wikimedia.org/r/173499
Change subject: Preloading tests
......................................................................
Preloading tests
QueryGenerator only performs continuation for the first
query module that uses continuation.
APISite.preloadpages always uses the 'revisions' module,
which needs continuation. The langlinks and templates
parameters of APISite.preloadpages require two additional
continuations to be operational at the same time, but this
exposes functionality that does not appear to be supported by
QueryGenerator.
There was only one very simple test for preloadpages. This
adds more tests for basic single continuation operation,
and three tests for multi-continuation operation.
Bug: 73461
Change-Id: I11df5361bd2b67cce132faba252552bfa5053827
---
M pywikibot/site.py
M tests/site_tests.py
2 files changed, 254 insertions(+), 16 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/pywikibot/core
refs/changes/99/173499/1
diff --git a/pywikibot/site.py b/pywikibot/site.py
index ba1b00c..814c9a6 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -2436,8 +2436,8 @@
break
else:
pywikibot.warning(
- u"preloadpages: Query returned unexpected
title"
- u"'%s'" % pagedata['title'])
+ u"preloadpages: Query returned unexpected "
+ u"title '%s'" % pagedata['title'])
continue
except KeyError:
pywikibot.debug(u"No 'title' in %s" % pagedata, _logger)
diff --git a/tests/site_tests.py b/tests/site_tests.py
index 10d36ed..edc6baa 100644
--- a/tests/site_tests.py
+++ b/tests/site_tests.py
@@ -280,20 +280,6 @@
if a:
self.assertEqual(a[0], mainpage)
- def testPreload(self):
- """Test that preloading works."""
- mysite = self.get_site()
- mainpage = self.get_mainpage()
- count = 0
- for page in mysite.preloadpages(mysite.pagelinks(mainpage, total=10)):
- self.assertIsInstance(page, pywikibot.Page)
- self.assertIsInstance(page.exists(), bool)
- if page.exists():
- self.assertTrue(hasattr(page, "_text"))
- count += 1
- if count >= 5:
- break
-
def testLinkMethods(self):
"""Test site methods for getting links to and from a page."""
mysite = self.get_site()
@@ -1562,6 +1548,258 @@
self.assertTrue(site.is_uploaddisabled())
+class TestPagePreloading(DefaultSiteTestCase):
+
+ """Test site.preloadpages()."""
+
+ def test_pageids(self):
+ """Test basic preloading with pageids."""
+ mysite = self.get_site()
+ mainpage = self.get_mainpage()
+ count = 0
+ links = mysite.pagelinks(mainpage, total=10)
+ # preloadpages will send the page ids,
+ # as they have already been loaded by pagelinks
+ for page in mysite.preloadpages(links):
+ self.assertIsInstance(page, pywikibot.Page)
+ self.assertIsInstance(page.exists(), bool)
+ if page.exists():
+ self.assertTrue(hasattr(page, "_text"))
+ self.assertEqual(len(page._revisions), 1)
+ self.assertFalse(hasattr(page, '_pageprops'))
+ count += 1
+ if count >= 5:
+ break
+
+ def test_titles(self):
+ """Test basic preloading with titles."""
+ mysite = self.get_site()
+ mainpage = self.get_mainpage()
+ count = 0
+ links = mysite.pagelinks(mainpage, total=10)
+
+ # remove the pageids that have already been loaded above by pagelinks
+ # so that preloadpages will use the titles instead
+ for page in links:
+ del page._pageid
+
+ for page in mysite.preloadpages(links):
+ self.assertIsInstance(page, pywikibot.Page)
+ self.assertIsInstance(page.exists(), bool)
+ if page.exists():
+ self.assertTrue(hasattr(page, "_text"))
+ self.assertEqual(len(page._revisions), 1)
+ self.assertFalse(hasattr(page, '_pageprops'))
+ count += 1
+ if count >= 5:
+ break
+
+ def test_preload_continuation(self):
+ """Test preloading continuation works."""
+ mysite = self.get_site()
+ mainpage = self.get_mainpage()
+ count = 0
+ links = mysite.pagelinks(mainpage, total=10)
+ for page in mysite.preloadpages(links, groupsize=5):
+ self.assertIsInstance(page, pywikibot.Page)
+ self.assertIsInstance(page.exists(), bool)
+ if page.exists():
+ self.assertTrue(hasattr(page, "_text"))
+ self.assertEqual(len(page._revisions), 1)
+ self.assertFalse(hasattr(page, '_pageprops'))
+ count += 1
+ if count >= 6:
+ break
+
+ def test_preload_high_groupsize(self):
+ """Test preloading continuation with groupsize greater than total."""
+ mysite = self.get_site()
+ mainpage = self.get_mainpage()
+ count = 0
+
+ # Determine if there are enough links on the main page,
+ # for the test to be useful.
+ link_count = len(list(mysite.pagelinks(mainpage, total=10)))
+ if link_count < 2:
+ raise unittest.SkipTest('insufficient links on main page')
+
+ # get a fresh generator; we now know how many results it will have,
+ # if it is less than 10.
+ links = mysite.pagelinks(mainpage, total=10)
+ for page in mysite.preloadpages(links, groupsize=50):
+ self.assertIsInstance(page, pywikibot.Page)
+ self.assertIsInstance(page.exists(), bool)
+ if page.exists():
+ self.assertTrue(hasattr(page, "_text"))
+ self.assertEqual(len(page._revisions), 1)
+ self.assertFalse(hasattr(page, '_pageprops'))
+ count += 1
+ self.assertEqual(count, link_count)
+
+ def test_preload_low_groupsize(self):
+ """Test preloading continuation with groupsize greater than total."""
+ mysite = self.get_site()
+ mainpage = self.get_mainpage()
+ count = 0
+
+ # Determine if there are enough links on the main page,
+ # for the test to be useful.
+ link_count = len(list(mysite.pagelinks(mainpage, total=10)))
+ if link_count < 2:
+ raise unittest.SkipTest('insufficient links on main page')
+
+ # get a fresh generator; we now know how many results it will have,
+ # if it is less than 10.
+ links = mysite.pagelinks(mainpage, total=10)
+ for page in mysite.preloadpages(links, groupsize=5):
+ self.assertIsInstance(page, pywikibot.Page)
+ self.assertIsInstance(page.exists(), bool)
+ if page.exists():
+ self.assertTrue(hasattr(page, "_text"))
+ self.assertEqual(len(page._revisions), 1)
+ self.assertFalse(hasattr(page, '_pageprops'))
+ count += 1
+ self.assertEqual(count, link_count)
+
+ def test_preload_unexpected_titles_using_pageids(self):
+ """Test sending pageids with unnormalized titles, causing warnings."""
+ mysite = self.get_site()
+ mainpage = self.get_mainpage()
+ count = 0
+ links = list(mysite.pagelinks(mainpage, total=10))
+ if len(links) < 2:
+ raise unittest.SkipTest('insufficient links on main page')
+
+ # change the title of the page, to test sametitle().
+ # preloadpages will send the page ids, as they have already been loaded
+ # by pagelinks, and preloadpages should complain the returned titles
+ # do not match any title in the pagelist.
+ for page in links:
+ page._link._text += ' '
+
+ gen = mysite.preloadpages(links, groupsize=5)
+ for page in gen:
+ self.assertIsInstance(page, pywikibot.Page)
+ self.assertIsInstance(page.exists(), bool)
+ if page.exists():
+ self.assertFalse(hasattr(page, "_text"))
+ self.assertEqual(len(page._revisions), 0)
+ self.assertFalse(hasattr(page, '_pageprops'))
+ count += 1
+ if count > 5:
+ break
+
+ def test_preload_unexpected_titles_using_titles(self):
+ """Test sending unnormalized titles, causing warnings."""
+ mysite = self.get_site()
+ mainpage = self.get_mainpage()
+ count = 0
+ links = list(mysite.pagelinks(mainpage, total=10))
+ if len(links) < 2:
+ raise unittest.SkipTest('insufficient links on main page')
+
+ # change the title of the page _and_ delete the pageids.
+ # preloadpages can only send the titles, and preloadpages should
+ # complain the returned titles do not match any title in the pagelist.
+ for page in links:
+ page._link._text += ' '
+ del page._pageid
+
+ gen = mysite.preloadpages(links, groupsize=5)
+ for page in gen:
+ self.assertIsInstance(page, pywikibot.Page)
+ self.assertIsInstance(page.exists(), bool)
+ if page.exists():
+ self.assertFalse(hasattr(page, "_text"))
+ self.assertEqual(len(page._revisions), 0)
+ self.assertFalse(hasattr(page, '_pageprops'))
+ count += 1
+ if count > 5:
+ break
+
+ def test_preload_invalid_titles_without_pageids(self):
+ """Test sending invalid titles. No warnings issued, but it should."""
+ mysite = self.get_site()
+ mainpage = self.get_mainpage()
+ count = 0
+ links = list(mysite.pagelinks(mainpage, total=10))
+ if len(links) < 2:
+ raise unittest.SkipTest('insufficient links on main page')
+
+ for page in links:
+ page._link._text += ' foobar'
+ del page._pageid
+
+ gen = mysite.preloadpages(links, groupsize=5)
+ for page in gen:
+ self.assertIsInstance(page, pywikibot.Page)
+ self.assertIsInstance(page.exists(), bool)
+ self.assertFalse(page.exists())
+ count += 1
+ if count > 5:
+ break
+
+ @unittest.expectedFailure
+ def test_preload_langlinks(self):
+ """Test preloading continuation works."""
+ mysite = self.get_site()
+ mainpage = self.get_mainpage()
+ count = 0
+ links = mysite.pagelinks(mainpage, total=10)
+ for page in mysite.preloadpages(links, groupsize=5, langlinks=True):
+ self.assertIsInstance(page, pywikibot.Page)
+ self.assertIsInstance(page.exists(), bool)
+ if page.exists():
+ self.assertTrue(hasattr(page, "_text"))
+ self.assertEqual(len(page._revisions), 1)
+ self.assertFalse(hasattr(page, '_pageprops'))
+ self.assertTrue(hasattr(page, '_langlinks'))
+ count += 1
+ if count >= 6:
+ break
+
+ @unittest.expectedFailure
+ def test_preload_templates(self):
+ """Test preloading templates works."""
+ mysite = self.get_site()
+ mainpage = self.get_mainpage()
+ count = 0
+ # Use backlinks, as any backlink has at least one link
+ links = mysite.pagelinks(mainpage, total=10)
+ for page in mysite.preloadpages(links, templates=True):
+ self.assertIsInstance(page, pywikibot.Page)
+ self.assertIsInstance(page.exists(), bool)
+ if page.exists():
+ self.assertTrue(hasattr(page, "_text"))
+ self.assertEqual(len(page._revisions), 1)
+ self.assertFalse(hasattr(page, '_pageprops'))
+ self.assertTrue(hasattr(page, '_templates'))
+ count += 1
+ if count >= 6:
+ break
+
+ @unittest.expectedFailure
+ def test_preload_templates_and_langlinks(self):
+ """Test preloading templates and langlinks works."""
+ mysite = self.get_site()
+ mainpage = self.get_mainpage()
+ count = 0
+ # Use backlinks, as any backlink has at least one link
+ links = mysite.pagebacklinks(mainpage, total=10)
+ for page in mysite.preloadpages(links, langlinks=True, templates=True):
+ self.assertIsInstance(page, pywikibot.Page)
+ self.assertIsInstance(page.exists(), bool)
+ if page.exists():
+ self.assertTrue(hasattr(page, "_text"))
+ self.assertEqual(len(page._revisions), 1)
+ self.assertFalse(hasattr(page, '_pageprops'))
+ self.assertTrue(hasattr(page, '_templates'))
+ self.assertTrue(hasattr(page, '_langlinks'))
+ count += 1
+ if count >= 6:
+ break
+
+
class TestDataSitePreloading(WikidataTestCase):
"""Test DataSite.preloaditempages for repo pages."""
--
To view, visit https://gerrit.wikimedia.org/r/173499
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I11df5361bd2b67cce132faba252552bfa5053827
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: John Vandenberg <jay...@gmail.com>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
