jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/288919 )

Change subject: [IMPR] Provide preloading via GeneratorFactory.getCombinedGenerator() ......................................................................

[IMPR] Provide preloading via GeneratorFactory.getCombinedGenerator()

- replace PreloadingGenerator if possible - also disable PreloadingGenerator when -liverecentchanges is selected

Bug: T135331 Change-Id: I52777a8b2405686ad7a3dffdf631789752afb7ff --- M pywikibot/pagegenerators.py M scripts/basic.py M scripts/blockpageschecker.py M scripts/capitalize_redirects.py M scripts/commons_link.py M scripts/commonscat.py M scripts/coordinate_import.py M scripts/cosmetic_changes.py M scripts/harvest_template.py M scripts/illustrate_wikidata.py M scripts/imagecopy.py M scripts/imagecopy_self.py M scripts/imageuncat.py M scripts/interwikidata.py M scripts/isbn.py M scripts/movepages.py M scripts/newitem.py M scripts/piper.py M scripts/reflinks.py M scripts/replace.py M scripts/selflink.py M scripts/table2wiki.py M scripts/template.py M scripts/touch.py M scripts/weblinkchecker.py 25 files changed, 95 insertions(+), 95 deletions(-)

Approvals: Dalba: Looks good to me, approved jenkins-bot: Verified

diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index d5a56f0..37936bc 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -394,6 +394,7 @@ self._site = site self._positional_arg_name = positional_arg_name self._sparql = None + self.nopreload = False

@property def site(self): @@ -436,10 +437,16 @@ self.site.namespaces.resolve(self._namespaces)) return self._namespaces

- def getCombinedGenerator(self, gen=None): + def getCombinedGenerator(self, gen=None, preload=False): """Return the combination of all accumulated generators.

Only call this after all arguments have been parsed. + + @param gen: Another generator to be combined with + @type gen: iterator + @param preload: preload pages using PreloadingGenerator + unless self.nopreload is True + @type preload: bool """ if gen: self.gens.insert(0, gen) @@ -509,6 +516,12 @@ if self.catfilter_list: dupfiltergen = CategoryFilterPageGenerator( dupfiltergen, self.catfilter_list, self.site) + + if preload and not self.nopreload: + if isinstance(dupfiltergen, DequeGenerator): + dupfiltergen = DequePreloadingGenerator(dupfiltergen) + else: + dupfiltergen = PreloadingGenerator(dupfiltergen)

return dupfiltergen

@@ -740,7 +753,9 @@ _filter_unique=self._filter_unique)

elif arg == '-liverecentchanges': + self.nopreload = True gen = LiveRCPageGenerator(site=self.site, total=intNone(value)) + elif arg == '-file': if not value: value = pywikibot.input('Please enter the local file name:') diff --git a/scripts/basic.py b/scripts/basic.py index 0f3879b..d056f38 100755 --- a/scripts/basic.py +++ b/scripts/basic.py @@ -25,7 +25,7 @@ -summary: Set the action summary message for the edit. """ # -# (C) Pywikibot team, 2006-2016 +# (C) Pywikibot team, 2006-2017 # # Distributed under the terms of the MIT license. # @@ -189,11 +189,10 @@ else: options[option] = True

- gen = genFactory.getCombinedGenerator() + # The preloading option is responsible for downloading multiple + # pages from the wiki simultaneously. + gen = genFactory.getCombinedGenerator(preload=True) if gen: - # The preloading generator is responsible for downloading multiple - # pages from the wiki simultaneously. - gen = pagegenerators.PreloadingGenerator(gen) # pass generator and private options to the bot bot = BasicBot(gen, **options) bot.run() # guess what it does diff --git a/scripts/blockpageschecker.py b/scripts/blockpageschecker.py index acacada..5fe9446 100755 --- a/scripts/blockpageschecker.py +++ b/scripts/blockpageschecker.py @@ -50,7 +50,7 @@ # (C) Monobi a.k.a. Wikihermit, 2007 # (C) Filnik, 2007-2011 # (C) Nicolas Dumazet (NicDumZ), 2008-2009 -# (C) Pywikibot team, 2007-2016 +# (C) Pywikibot team, 2007-2017 # # Distributed under the terms of the MIT license. # @@ -284,8 +284,10 @@ generator.append(pageCat) pywikibot.output(u'Categories loaded, start!') # Main Loop - preloadingGen = pagegenerators.PreloadingGenerator(generator, groupsize=60) - for page in preloadingGen: + if not genFactory.nopreload: + generator = pagegenerators.PreloadingGenerator(generator, + groupsize=60) + for page in generator: pagename = page.title(asLink=True) pywikibot.output('Loading %s...' % pagename) try: diff --git a/scripts/capitalize_redirects.py b/scripts/capitalize_redirects.py index 7a7d75d..6576e9e 100755 --- a/scripts/capitalize_redirects.py +++ b/scripts/capitalize_redirects.py @@ -22,7 +22,7 @@ """ # # (C) Yrithinnd, 2006 -# (C) Pywikibot team, 2007-2015 +# (C) Pywikibot team, 2007-2017 # # Distributed under the terms of the MIT license. # @@ -111,10 +111,9 @@ else: genFactory.handleArg(arg)

- gen = genFactory.getCombinedGenerator() + gen = genFactory.getCombinedGenerator(preload=True) if gen: - preloadingGen = pagegenerators.PreloadingGenerator(gen) - bot = CapitalizeBot(preloadingGen, **options) + bot = CapitalizeBot(gen, **options) bot.run() return True else: diff --git a/scripts/commons_link.py b/scripts/commons_link.py index 925cdd1..3ddffaf 100755 --- a/scripts/commons_link.py +++ b/scripts/commons_link.py @@ -24,7 +24,7 @@ """ # # (C) Leonardo Gregianin, 2006 -# (C) Pywikibot team, 2007-2016 +# (C) Pywikibot team, 2007-2017 # # Distributed under the terms of the MIT license. # @@ -140,9 +140,8 @@ else: genFactory.handleArg(arg)

- gen = genFactory.getCombinedGenerator() + gen = genFactory.getCombinedGenerator(preload=True) if 'action' in options and gen: - gen = pagegenerators.PreloadingGenerator(gen) bot = CommonsLinkBot(gen, **options) bot.run() return True diff --git a/scripts/commonscat.py b/scripts/commonscat.py index 44a62d7..fc710ae 100755 --- a/scripts/commonscat.py +++ b/scripts/commonscat.py @@ -52,8 +52,8 @@ # November 2013 # # (C) Multichill, 2008-2009 -# (C) Xqt, 2009-2015 -# (C) Pywikibot team, 2008-2015 +# (C) Xqt, 2009-2017 +# (C) Pywikibot team, 2008-2017 # # Distributed under the terms of the MIT license. # @@ -536,8 +536,9 @@ generator = genFactory.getCombinedGenerator()

if generator: - pregenerator = pagegenerators.PreloadingGenerator(generator) - bot = CommonscatBot(pregenerator, **options) + if not genFactory.nopreload: + generator = pagegenerators.PreloadingGenerator(generator) + bot = CommonscatBot(generator, **options) bot.run() return True else: diff --git a/scripts/coordinate_import.py b/scripts/coordinate_import.py index 7ed163d..9000b31 100755 --- a/scripts/coordinate_import.py +++ b/scripts/coordinate_import.py @@ -25,7 +25,7 @@ """ # # (C) Multichill, 2014 -# (C) Pywikibot team, 2013-2016 +# (C) Pywikibot team, 2013-2017 # # Distributed under the terms of MIT License. # @@ -49,7 +49,7 @@ @param generator: A generator that yields Page objects. """ super(CoordImportRobot, self).__init__() - self.generator = pagegenerators.PreloadingGenerator(generator) + self.generator = generator self.cacheSources() self.prop = 'P625'

@@ -123,7 +123,7 @@ if generator_factory.handleArg(arg): continue

- generator = generator_factory.getCombinedGenerator() + generator = generator_factory.getCombinedGenerator(preload=True)

if generator: coordbot = CoordImportRobot(generator) diff --git a/scripts/cosmetic_changes.py b/scripts/cosmetic_changes.py index 9117d5a..bc90df3 100644 --- a/scripts/cosmetic_changes.py +++ b/scripts/cosmetic_changes.py @@ -26,7 +26,7 @@ For further information see pywikibot/cosmetic_changes.py """ # -# (C) xqt, 2009-2013 +# (C) xqt, 2009-2017 # (C) Pywikibot team, 2006-2017 # # Distributed under the terms of the MIT license. @@ -123,14 +123,13 @@ options['summary'] = i18n.twtranslate(site, 'cosmetic_changes-standalone')

- gen = genFactory.getCombinedGenerator() + gen = genFactory.getCombinedGenerator(preload=True) if gen: if options.get('always') or config.simulate or pywikibot.input_yn( warning + '\nDo you really want to continue?', default=False, automatic_quit=False): site.login() - preloadingGen = pagegenerators.PreloadingGenerator(gen) - bot = CosmeticChangesBot(preloadingGen, **options) + bot = CosmeticChangesBot(gen, **options) bot.run() return True else: diff --git a/scripts/harvest_template.py b/scripts/harvest_template.py index 62d24c6..653c101 100755 --- a/scripts/harvest_template.py +++ b/scripts/harvest_template.py @@ -72,7 +72,7 @@

""" super(HarvestRobot, self).__init__() - self.generator = pg.PreloadingGenerator(generator) + self.generator = generator self.templateTitle = templateTitle.replace(u'_', u' ') # TODO: Make it a list which also includes the redirects to the template self.fields = fields @@ -262,10 +262,10 @@ for i in range(0, len(commandline_arguments), 2): fields[commandline_arguments[i]] = commandline_arguments[i + 1]

- generator = gen.getCombinedGenerator() + generator = gen.getCombinedGenerator(preload=True) if not generator: gen.handleArg(u'-transcludes:' + template_title) - generator = gen.getCombinedGenerator() + generator = gen.getCombinedGenerator(preload=True)

bot = HarvestRobot(generator, template_title, fields) bot.run() diff --git a/scripts/illustrate_wikidata.py b/scripts/illustrate_wikidata.py index 6e0f91a..c48eef5 100755 --- a/scripts/illustrate_wikidata.py +++ b/scripts/illustrate_wikidata.py @@ -15,7 +15,7 @@ """ # # (C) Multichill, 2014 -# (C) Pywikibot team, 2013-2016 +# (C) Pywikibot team, 2013-2017 # # Distributed under the terms of MIT License. # @@ -45,7 +45,7 @@ @type wdproperty: str """ super(IllustrateRobot, self).__init__() - self.generator = pagegenerators.PreloadingGenerator(generator) + self.generator = generator self.wdproperty = wdproperty self.cacheSources()

@@ -119,7 +119,7 @@ else: generator_factory.handleArg(arg)

- generator = generator_factory.getCombinedGenerator() + generator = generator_factory.getCombinedGenerator(preload=True) if not generator: pywikibot.bot.suggest_help(missing_generator=True) return False diff --git a/scripts/imagecopy.py b/scripts/imagecopy.py index 49ce90b..5a5700a 100644 --- a/scripts/imagecopy.py +++ b/scripts/imagecopy.py @@ -77,7 +77,7 @@ # # Another rewrite by: # (C) Multichill 2008-2011 -# (C) Pywikibot team, 2007-2016 +# (C) Pywikibot team, 2007-2017 # # Distributed under the terms of the MIT license. # @@ -494,7 +494,6 @@

def main(*args): """Process command line arguments and invoke bot.""" - generator = None imagepage = None always = False category = u'' @@ -510,12 +509,10 @@ else: genFactory.handleArg(arg)

- generator = genFactory.getCombinedGenerator() - if not generator: + pregenerator = genFactory.getCombinedGenerator(preload=True) + if not pregenerator: pywikibot.bot.suggest_help(missing_generator=True) return False - - pregenerator = pagegenerators.PreloadingGenerator(generator)

for page in pregenerator: skip = False diff --git a/scripts/imagecopy_self.py b/scripts/imagecopy_self.py index 0f2e12f..32e7954 100644 --- a/scripts/imagecopy_self.py +++ b/scripts/imagecopy_self.py @@ -45,7 +45,7 @@ # English Wikipedia specific bot by: # (C) Multichill 2010-2012 # -# (C) Pywikibot team, 2010-2016 +# (C) Pywikibot team, 2010-2017 # # Distributed under the terms of the MIT license. # @@ -1022,7 +1022,6 @@

def main(*args): """Process command line arguments and invoke bot.""" - generator = None autonomous = False checkTemplate = True

@@ -1037,8 +1036,8 @@ else: genFactory.handleArg(arg)

- generator = genFactory.getCombinedGenerator() - if not generator: + pregenerator = genFactory.getCombinedGenerator(preload=True) + if not pregenerator: pywikibot.bot.suggest_help(missing_generator=True) return False

@@ -1050,8 +1049,6 @@ pywikibot.warning(u'It will only work on self published work images') pywikibot.warning(u'This bot is still full of bugs') pywikibot.warning(u'Use at your own risk!') - - pregenerator = pagegenerators.PreloadingGenerator(generator)

prefetchQueue = Queue(maxsize=50) uploadQueue = Queue(maxsize=200) diff --git a/scripts/imageuncat.py b/scripts/imageuncat.py index 2e3693c..7bb9db1 100755 --- a/scripts/imageuncat.py +++ b/scripts/imageuncat.py @@ -1366,14 +1366,13 @@ else: genFactory.handleArg(arg)

- generator = genFactory.getCombinedGenerator(gen=generator) + generator = genFactory.getCombinedGenerator(gen=generator, preload=True) if not generator: pywikibot.bot.suggest_help(missing_generator=True) return False else: - pregenerator = pagegenerators.PreloadingGenerator(generator) site.login() - for page in pregenerator: + for page in generator: pywikibot.output(page.title()) if page.exists() and (page.namespace() == 6) \ and (not page.isRedirectPage()): diff --git a/scripts/interwikidata.py b/scripts/interwikidata.py index 63dce6e..e2fea57 100644 --- a/scripts/interwikidata.py +++ b/scripts/interwikidata.py @@ -21,7 +21,7 @@ -summary: Use your own edit summary for cleaning the page. """

-# (C) Pywikibot team, 2015-2016 +# (C) Pywikibot team, 2015-2017 # # Distributed under the terms of the MIT license. # @@ -197,9 +197,8 @@

site = pywikibot.Site()

- generator = genFactory.getCombinedGenerator() + generator = genFactory.getCombinedGenerator(preload=True) if generator: - generator = pagegenerators.PreloadingGenerator(generator) bot = IWBot(generator=generator, site=site, **options) bot.run() else: diff --git a/scripts/isbn.py b/scripts/isbn.py index cfd1fbb..415df16 100755 --- a/scripts/isbn.py +++ b/scripts/isbn.py @@ -35,7 +35,7 @@

""" # -# (C) Pywikibot team, 2009-2015 +# (C) Pywikibot team, 2009-2017 # # Distributed under the terms of the MIT license. # @@ -1662,13 +1662,12 @@ else: genFactory.handleArg(arg)

- gen = genFactory.getCombinedGenerator() + gen = genFactory.getCombinedGenerator(preload=True) if gen: - preloadingGen = pagegenerators.PreloadingGenerator(gen) if use_wikibase: - bot = IsbnWikibaseBot(preloadingGen, **options) + bot = IsbnWikibaseBot(gen, **options) else: - bot = IsbnBot(preloadingGen, **options) + bot = IsbnBot(gen, **options) bot.run() return True else: diff --git a/scripts/movepages.py b/scripts/movepages.py index 5a61392..b4ad5bd 100755 --- a/scripts/movepages.py +++ b/scripts/movepages.py @@ -34,7 +34,7 @@ # # (C) Leonardo Gregianin, 2006 # (C) Andreas J. Schwab, 2007 -# (C) Pywikibot team, 2006-2016 +# (C) Pywikibot team, 2006-2017 # # Distributed under the terms of the MIT license. # @@ -183,7 +183,6 @@ @param args: command line arguments @type args: list of unicode """ - gen = None oldName = None options = {} fromToPairs = [] @@ -253,11 +252,9 @@ bot = MovePagesBot(None, **options) bot.moveOne(page, pair[1])

- if not gen: - gen = genFactory.getCombinedGenerator() + gen = genFactory.getCombinedGenerator(preload=True) if gen: - preloadingGen = pagegenerators.PreloadingGenerator(gen) - bot = MovePagesBot(preloadingGen, **options) + bot = MovePagesBot(gen, **options) bot.run() return True

diff --git a/scripts/newitem.py b/scripts/newitem.py index 9d83d45..401de40 100755 --- a/scripts/newitem.py +++ b/scripts/newitem.py @@ -20,7 +20,7 @@ """ # # (C) Multichill, 2014 -# (C) Pywikibot team, 2014-2016 +# (C) Pywikibot team, 2014-2017 # # Distributed under the terms of the MIT license. # @@ -48,7 +48,7 @@ })

super(NewItemRobot, self).__init__(**kwargs) - self.generator = pagegenerators.PreloadingGenerator(generator) + self.generator = generator self.pageAge = self.getOption('pageage') self.lastEdit = self.getOption('lastedit') self.pageAgeBefore = self.repo.getcurrenttime() - timedelta( @@ -160,7 +160,7 @@ else: options[arg[1:].lower()] = True

- generator = gen.getCombinedGenerator() + generator = gen.getCombinedGenerator(preload=True) if not generator: pywikibot.bot.suggest_help(missing_generator=True) return False diff --git a/scripts/piper.py b/scripts/piper.py index d15e4c8..757d4b5 100755 --- a/scripts/piper.py +++ b/scripts/piper.py @@ -142,11 +142,10 @@

options['filters'] = filters

- gen = genFactory.getCombinedGenerator() + gen = genFactory.getCombinedGenerator(preload=True) if gen: # The preloading generator is responsible for downloading multiple # pages from the wiki simultaneously. - gen = pagegenerators.PreloadingGenerator(gen) bot = PiperBot(gen, **options) bot.run() return True diff --git a/scripts/reflinks.py b/scripts/reflinks.py index fd3352f..44e9d10 100755 --- a/scripts/reflinks.py +++ b/scripts/reflinks.py @@ -38,7 +38,7 @@ one from i18n/reflinks.py """ # (C) Nicolas Dumazet (NicDumZ), 2008 -# (C) Pywikibot team, 2008-2016 +# (C) Pywikibot team, 2008-2017 # # Distributed under the terms of the MIT license. # @@ -786,8 +786,8 @@ if not generator: pywikibot.bot.suggest_help(missing_generator=True) return False - - generator = pagegenerators.PreloadingGenerator(generator) + if not genFactory.nopreload: + generator = pagegenerators.PreloadingGenerator(generator) generator = pagegenerators.RedirectFilterPageGenerator(generator) bot = ReferencesRobot(generator, **options) bot.run() diff --git a/scripts/replace.py b/scripts/replace.py index f235c95..66a15da 100755 --- a/scripts/replace.py +++ b/scripts/replace.py @@ -1142,14 +1142,13 @@ LIMIT 200""" % (whereClause, exceptClause) gen = pagegenerators.MySQLPageGenerator(query)

- gen = genFactory.getCombinedGenerator(gen) + gen = genFactory.getCombinedGenerator(gen, preload=True)

if not gen: pywikibot.bot.suggest_help(missing_generator=True) return False

- preloadingGen = pagegenerators.PreloadingGenerator(gen) - bot = ReplaceRobot(preloadingGen, replacements, exceptions, + bot = ReplaceRobot(gen, replacements, exceptions, allowoverlap, recursive, add_cat, sleep, edit_summary, always=acceptall, site=site) site.login() diff --git a/scripts/selflink.py b/scripts/selflink.py index 116b6a1..69a728f 100755 --- a/scripts/selflink.py +++ b/scripts/selflink.py @@ -11,7 +11,7 @@ ATTENTION: Use this with care! """ # -# (C) Pywikibot team, 2006-2015 +# (C) Pywikibot team, 2006-2017 # # Distributed under the terms of the MIT license. # @@ -20,8 +20,7 @@ import pywikibot

from pywikibot.bot import Choice, MultipleSitesBot -from pywikibot.pagegenerators import GeneratorFactory, PreloadingGenerator, \ - parameterHelp +from pywikibot.pagegenerators import GeneratorFactory, parameterHelp

from scripts.unlink import BaseUnlinkBot

@@ -96,13 +95,12 @@ else: genFactory.handleArg(arg)

- gen = genFactory.getCombinedGenerator() + gen = genFactory.getCombinedGenerator(preload=True) if not gen: pywikibot.bot.suggest_help(missing_generator=True) return False

- preloadingGen = PreloadingGenerator(gen) - bot = SelflinkBot(preloadingGen, **botArgs) + bot = SelflinkBot(gen, **botArgs) bot.run() return True

diff --git a/scripts/table2wiki.py b/scripts/table2wiki.py index 87aa6dc..7559ee5 100644 --- a/scripts/table2wiki.py +++ b/scripts/table2wiki.py @@ -48,7 +48,7 @@ """ # # (C) 2003 Thomas R. Koll, tomk32@tomk32.de -# (C) Pywikibot team, 2003-2013 +# (C) Pywikibot team, 2003-2017 # # Distributed under the terms of the MIT license. # @@ -594,8 +594,9 @@ if gen: if namespaces != []: gen = pagegenerators.NamespaceFilterPageGenerator(gen, namespaces) - preloadingGen = pagegenerators.PreloadingGenerator(gen) - bot = Table2WikiRobot(preloadingGen, quietMode) + if not genFactory.nopreload: + gen = pagegenerators.PreloadingGenerator(gen) + bot = Table2WikiRobot(gen, quietMode) bot.run() else: pywikibot.showHelp('table2wiki') diff --git a/scripts/template.py b/scripts/template.py index b4bd2aa..7ccffa3 100755 --- a/scripts/template.py +++ b/scripts/template.py @@ -105,8 +105,8 @@ # # (C) Daniel Herding, 2004 # (C) Rob W.W. Hooft, 2003-2005 -# (C) xqt, 2009-2016 -# (C) Pywikibot team, 2004-2016 +# (C) xqt, 2009-2017 +# (C) Pywikibot team, 2004-2017 # # Distributed under the terms of the MIT license. # @@ -356,9 +356,10 @@ # make sure that proper namespace filtering etc. is handled gen = genFactory.getCombinedGenerator(gen)

- preloadingGen = pagegenerators.PreloadingGenerator(gen) + if not genFactory.nopreload: + gen = pagegenerators.PreloadingGenerator(gen)

- bot = TemplateRobot(preloadingGen, templates, site=site, **options) + bot = TemplateRobot(gen, templates, site=site, **options) bot.run()

if __name__ == "__main__": diff --git a/scripts/touch.py b/scripts/touch.py index 87a9c61..88d8b5f 100755 --- a/scripts/touch.py +++ b/scripts/touch.py @@ -15,7 +15,7 @@

""" # -# (C) Pywikibot team, 2009-2015 +# (C) Pywikibot team, 2009-2017 # # Distributed under the terms of the MIT license. # @@ -97,10 +97,9 @@ # -botflag options[arg[1:].lower()] = True

- gen = genFactory.getCombinedGenerator() + gen = genFactory.getCombinedGenerator(preload=True) if gen: - preloadingGen = pagegenerators.PreloadingGenerator(gen) - bot = bot_class(generator=preloadingGen, **options) + bot = bot_class(generator=gen, **options) pywikibot.Site().login() bot.run() return True diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py index b8cb323..9fd0274 100755 --- a/scripts/weblinkchecker.py +++ b/scripts/weblinkchecker.py @@ -95,7 +95,7 @@ """ # # (C) Daniel Herding, 2005 -# (C) Pywikibot team, 2005-2016 +# (C) Pywikibot team, 2005-2017 # # Distributed under the terms of the MIT license. # @@ -976,10 +976,11 @@ if not gen: gen = genFactory.getCombinedGenerator() if gen: - # fetch at least 240 pages simultaneously from the wiki, but more if - # a high thread number is set. - pageNumber = max(240, config.max_external_links * 2) - gen = pagegenerators.PreloadingGenerator(gen, groupsize=pageNumber) + if not genFactory.nopreload: + # fetch at least 240 pages simultaneously from the wiki, but more + # if a high thread number is set. + pageNumber = max(240, config.max_external_links * 2) + gen = pagegenerators.PreloadingGenerator(gen, groupsize=pageNumber) gen = pagegenerators.RedirectFilterPageGenerator(gen) bot = WeblinkCheckerRobot(gen, HTTPignore, config.weblink_dead_days) try: