jenkins-bot submitted this change.

View Change


Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified
[IMPR] Provide bulk purge for touch.py

The script is able to run upto 1000 times faster.

- instead of purge one page after another use a list of pages for
site.purgepages but take into account that purge action is rate
limited. Use the default rate limit which is 30/60 until a ratelimit
method is implemented
- do not preload pages when purging pages. Preloading is only usefull
when touching.
- use pywikibot.bot.suggest_help to check whether the bot should run

patch detached from If5b1334e66d2f648dbde4ceec2918f87097bb9f6

Change-Id: Ie60a4ebff59a337359f6b67e0e97bcfa15eb6fb8
---
M scripts/touch.py
1 file changed, 84 insertions(+), 13 deletions(-)

diff --git a/scripts/touch.py b/scripts/touch.py
index c3c447c..047fe23 100755
--- a/scripts/touch.py
+++ b/scripts/touch.py
@@ -23,12 +23,15 @@
&params;
"""
#
-# (C) Pywikibot team, 2009-2022
+# (C) Pywikibot team, 2009-2023
#
# Distributed under the terms of the MIT license.
#
+from collections import defaultdict
+from contextlib import suppress
+
import pywikibot
-from pywikibot import pagegenerators
+from pywikibot import config, pagegenerators
from pywikibot.bot import MultipleSitesBot
from pywikibot.exceptions import (
LockedPageError,
@@ -76,12 +79,57 @@
'redirects': None
}

+ def __init__(self, *args, **kwargs):
+ """Initializer."""
+ super().__init__(*args, **kwargs)
+ self.pages = defaultdict(list)
+
def treat(self, page) -> None:
- """Purge the given page."""
- done = page.purge(**self.opt)
- if done:
- self.counter['purge'] += 1
- pywikibot.info('Page {}{} purged'.format(page, '' if done else ' not'))
+ """Purge the given page.
+
+ .. versionchanged:: 8.0
+ Enable batch purge using :meth:`APISite.purgepages()
+ <pywikibot.site._apisite.APISite.purgepages>`
+ """
+ # We can have mutiple sites, save pages and cache rate limit
+ self.pages[page.site].append(page)
+ self.purgepages()
+
+ def teardown(self):
+ """Purge remaining pages if no KeyboardInterrupt was made.
+
+ .. versionadded:: 8.0
+ """
+ if self.generator_completed:
+ with suppress(KeyboardInterrupt):
+ self.purgepages(flush=True)
+
+ # show the counter even no purges were made
+ self.counter['purge'] += 0
+
+ def purgepages(self, flush=False):
+ """Purge a bulk of page if rate limit exceeded.
+
+ Use default rate limit for purging pages which is 30/60.
+
+ .. versionadded:: 8.0
+ """
+ for site, pagelist in self.pages.items():
+ length = len(pagelist)
+ if flush or length >= 30:
+ done = site.purgepages(pagelist, **self.opt)
+ if done:
+ self.counter['purge'] += length
+ self.pages[site].clear()
+
+ pywikibot.info('{} pages{} purged'
+ .format(length, '' if done else ' not'))
+ if not flush:
+ delay = 60 + 2
+ if delay:
+ pywikibot.info(
+ f'Waiting {delay} seconds due to purge rate limit')
+ pywikibot.sleep(0 if config.simulate else delay)


def main(*args: str) -> None:
@@ -93,6 +141,7 @@
:param args: command line arguments
"""
options = {}
+ unknown = []

# Process global and pagegenerators args
local_args = pywikibot.handle_args(args)
@@ -101,17 +150,18 @@

bot_class = TouchBot
for arg in local_args:
+ option, _, value = arg[1:].partition(':')
if arg == '-purge':
bot_class = PurgeBot
elif arg.startswith('-'):
- options[arg[1:].lower()] = True
+ options[option.lower()] = True
+ else:
+ unknown.append(arg)

- if gen_factory.gens:
- gen = gen_factory.getCombinedGenerator(preload=True)
- pywikibot.Site().login()
+ if not pywikibot.bot.suggest_help(missing_generator=not gen_factory.gens,
+ unknown_parameters=unknown):
+ gen = gen_factory.getCombinedGenerator(preload=bot_class == TouchBot)
bot_class(generator=gen, **options).run()
- else:
- pywikibot.bot.suggest_help(missing_generator=True)


if __name__ == '__main__':

To view, visit change 877187. To unsubscribe, or for help writing mail filters, visit settings.

Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ie60a4ebff59a337359f6b67e0e97bcfa15eb6fb8
Gerrit-Change-Number: 877187
Gerrit-PatchSet: 2
Gerrit-Owner: Xqt <info@gno.de>
Gerrit-Reviewer: D3r1ck01 <xsavitar.wiki@aol.com>
Gerrit-Reviewer: Xqt <info@gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged