jenkins-bot has submitted this change and it was merged.

Change subject: PreloadingItemGenerator: process normal pages ......................................................................

PreloadingItemGenerator: process normal pages

The item preloading generator only prefetches if the input generator yields WikibasePage, with a Site class that has method preloaditempages.

This allows a Page which is in the 'wikibase-item' namespace to be be preloaded.

Bug: 54999 Change-Id: I62140a91bb68a67a93dcf7bb70bfe284de672d02 --- M pywikibot/pagegenerators.py M tests/pagegenerators_tests.py M tests/wikibase_tests.py 3 files changed, 40 insertions(+), 1 deletion(-)

Approvals: XZise: Looks good to me, approved jenkins-bot: Verified

diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index 0ba02b4..2455532 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -1082,6 +1082,16 @@ """ sites = {} for page in generator: + if not isinstance(page, pywikibot.page.WikibasePage): + datasite = page.site.data_repository() + if page.namespace() != datasite.item_namespace: + pywikibot.output( + u'PreloadingItemGenerator skipping %s as it is not in %s' + % (page, datasite.item_namespace)) + continue + + page = pywikibot.ItemPage(datasite, page.title()) + site = page.site sites.setdefault(site, []).append(page) if len(sites[site]) >= step: diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py index 96f574c..f7f372c 100755 --- a/tests/pagegenerators_tests.py +++ b/tests/pagegenerators_tests.py @@ -10,7 +10,7 @@ import pywikibot from pywikibot import pagegenerators

-from tests.aspects import unittest, TestCase +from tests.aspects import unittest, TestCase, WikidataTestCase

class TestPageGenerators(TestCase): @@ -153,6 +153,19 @@ self.assertEqual(len(set(item['revid'] for item in items)), 4)

+class TestPreloadingItemGenerator(WikidataTestCase): + + """Test preloading item generator.""" + + def test_non_item_gen(self): + """Test TestPreloadingItemGenerator with ReferringPageGenerator.""" + site = self.get_site() + instance_of_page = pywikibot.Page(site, 'Property:P31') + ref_gen = pagegenerators.ReferringPageGenerator(instance_of_page, total=5) + gen = pagegenerators.PreloadingItemGenerator(ref_gen) + self.assertTrue(all(isinstance(item, pywikibot.ItemPage) for item in gen)) + + if __name__ == "__main__": try: unittest.main() diff --git a/tests/wikibase_tests.py b/tests/wikibase_tests.py index 7726433..6fce7df 100644 --- a/tests/wikibase_tests.py +++ b/tests/wikibase_tests.py @@ -631,6 +631,22 @@ self.assertEqual(response, self.data_out)

+class TestPreloadingItemGenerator(TestCase): + + """Test preloading item generator.""" + + family = 'wikidata' + code = 'wikidata' + + def test_non_item_gen(self): + """Test TestPreloadingItemGenerator with ReferringPageGenerator.""" + site = self.get_site() + instance_of_page = pywikibot.Page(site, 'Property:P31') + ref_gen = pagegenerators.ReferringPageGenerator(instance_of_page, total=5) + gen = pagegenerators.PreloadingItemGenerator(ref_gen) + self.assertTrue(all(isinstance(item, pywikibot.ItemPage) for item in gen)) + + class TestNamespaces(WikidataTestCase):

"""Test cases to test namespaces of Wikibase entities."""