jenkins-bot has submitted this change and it was merged.

Change subject: DequeGenerator ......................................................................

DequeGenerator

Scripts may be written to dynamically populate a list of pages to work on. Ideally this list would be created using a generator, however that can require a large rewrite in the flow control.

To simplify the porting of scripts, this changeset adds classes DequeGenerator and DequePreloadingGenerator classes that allow the page list to be extended during iteration.

Implemented in makecat.

Change-Id: I9e67a4d9d0f1736071434fa84796f22732896088 --- M pywikibot/pagegenerators.py M pywikibot/tools.py M scripts/makecat.py M tests/pagegenerators_tests.py 4 files changed, 72 insertions(+), 22 deletions(-)

Approvals: XZise: Looks good to me, approved jenkins-bot: Verified

diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index eb8930a..b79dfda 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -28,7 +28,7 @@ import time import pywikibot from pywikibot import date, config, i18n -from pywikibot.tools import deprecated_args +from pywikibot.tools import deprecated_args, DequeGenerator from pywikibot.comms import http import pywikibot.data.wikidataquery as wdquery

@@ -1143,6 +1143,18 @@ yield i

+def DequePreloadingGenerator(generator, step=50): + assert(isinstance(generator, DequeGenerator)) + + while True: + page_count = min(len(generator), step) + if not page_count: + raise StopIteration + + for page in PreloadingGenerator(generator, page_count): + yield page + + def PreloadingItemGenerator(generator, step=50): """ Yield preloaded pages taken from another generator. diff --git a/pywikibot/tools.py b/pywikibot/tools.py index 6064d86..dcf4fd4 100644 --- a/pywikibot/tools.py +++ b/pywikibot/tools.py @@ -13,7 +13,7 @@ import time import inspect import re -from collections import Mapping +from collections import Mapping, deque from distutils.version import Version

if sys.version_info[0] > 2: @@ -320,6 +320,26 @@

EMPTY_DEFAULT = EmptyDefault()

+ +class DequeGenerator(deque): + + """A generator that allows items to be added during generating.""" + + def __iter__(self): + """Return the object which will be iterated.""" + return self + + def next(self): + """Python 3 iterator method.""" + if len(self): + return self.popleft() + else: + raise StopIteration + + def __next__(self): + """Python 3 iterator method.""" + return self.next() + # Decorators # # Decorator functions without parameters are _invoked_ differently from diff --git a/scripts/makecat.py b/scripts/makecat.py index e523cff..42f4a7e 100644 --- a/scripts/makecat.py +++ b/scripts/makecat.py @@ -44,6 +44,7 @@ import codecs import pywikibot from pywikibot import date, pagegenerators, i18n, textlib +from pywikibot.tools import DequeGenerator

def isdate(s): @@ -197,7 +198,7 @@ removeparent = True main = True workingcatname = '' - tocheck = [] + tocheck = DequeGenerator() for arg in pywikibot.handleArgs(): if arg.startswith('-nodate'): skipdates = True @@ -271,26 +272,14 @@ answer = workingcatname pywikibot.output(u'' + answer) pl = pywikibot.Page(mysite, answer) - tocheck = [] checked[pl] = pl include(pl) - loaded = 0 - while tocheck: - if loaded == 0: - if len(tocheck) < 50: - loaded = len(tocheck) - else: - loaded = 50 - tocheck = [x for x in pagegenerators.PreloadingGenerator(tocheck[:loaded])] - if not checkbroken: - if not tocheck[0].exists(): - pass - else: - asktoadd(tocheck[0]) - else: - asktoadd(tocheck[0]) - tocheck = tocheck[1:] - loaded -= 1 + + gen = pagegenerators.DequePreloadingGenerator(tocheck) + + for page in gen: + if checkbroken or page.exists(): + asktoadd(page)

finally: try: diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py index 44a99cd..dd85f1b 100755 --- a/tests/pagegenerators_tests.py +++ b/tests/pagegenerators_tests.py @@ -11,7 +11,12 @@ import pywikibot from pywikibot import pagegenerators

-from tests.aspects import unittest, TestCase, WikidataTestCase +from tests.aspects import ( + unittest, + TestCase, + WikidataTestCase, + DefaultSiteTestCase, +)

class TestPageGenerators(TestCase): @@ -154,6 +159,30 @@ self.assertEqual(len(set(item['revid'] for item in items)), 4)

+class TestDequePreloadingGenerator(DefaultSiteTestCase): + + """Test preloading generator on lists.""" + + def test_deque_preloading(self): + """Test pages being added to a DequePreloadingGenerator.""" + mainpage = self.get_mainpage() + + pages = pywikibot.tools.DequeGenerator([mainpage]) + gen = pagegenerators.DequePreloadingGenerator(pages) + pages_out = list() + for page in gen: + pages_out.append(page) + # Add a page to the generator + if page.namespace() == 0: + pages.extend([page.toggleTalkPage()]) + + self.assertTrue(all(isinstance(page, pywikibot.Page) for page in pages_out)) + self.assertEqual(len(pages_out), 2) + self.assertEqual(pages_out[1].namespace(), 1) + self.assertIn(mainpage, pages_out) + self.assertIn(mainpage.toggleTalkPage(), pages_out) + + class TestPreloadingItemGenerator(WikidataTestCase):

"""Test preloading item generator."""