jenkins-bot submitted this change.

View Change


Approvals: JJMC89: Looks good to me, approved jenkins-bot: Verified
[IMPR] Use site.ratelimit for bulk purge in PurgeBot

Change-Id: If5b1334e66d2f648dbde4ceec2918f87097bb9f6
---
M scripts/touch.py
1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/scripts/touch.py b/scripts/touch.py
index 3fdfbf5..e0ca6c2 100755
--- a/scripts/touch.py
+++ b/scripts/touch.py
@@ -83,6 +83,7 @@
"""Initializer."""
super().__init__(*args, **kwargs)
self.pages = defaultdict(list)
+ self.limit = {}

def treat(self, page) -> None:
"""Purge the given page.
@@ -93,6 +94,7 @@
"""
# We can have mutiple sites, save pages and cache rate limit
self.pages[page.site].append(page)
+ self.limit.setdefault(page.site, page.site.ratelimit('purge'))
self.purgepages()

def teardown(self):
@@ -110,13 +112,15 @@
def purgepages(self, flush=False):
"""Purge a bulk of page if rate limit exceeded.

- Use default rate limit for purging pages which is 30/60.
-
.. versionadded:: 8.0
+ .. versionchanged:: 9.0
+ :meth:`site.APISite.ratelimit()
+ <pywikibot.site._apisite.APISite.ratelimit>` method is used
+ to determine bulk length and delay.
"""
for site, pagelist in self.pages.items():
length = len(pagelist)
- if flush or length >= 30:
+ if flush or length >= self.limit[site].hits:
done = site.purgepages(pagelist, **self.opt)
if done:
self.counter['purge'] += length
@@ -124,9 +128,12 @@

pywikibot.info(
f"{length} pages{'' if done else ' not'} purged")
- if not flush and not config.simulate:
- pywikibot.info('Waiting due to purge rate limit')
- pywikibot.sleep(62)
+ if not flush and config.simulate is False:
+ delay = self.limit[site].delay * (length + 1)
+ if delay:
+ pywikibot.info(
+ f'Waiting {delay} seconds due to purge rate limit')
+ pywikibot.sleep(delay)


def main(*args: str) -> None:

To view, visit change 834036. To unsubscribe, or for help writing mail filters, visit settings.

Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: If5b1334e66d2f648dbde4ceec2918f87097bb9f6
Gerrit-Change-Number: 834036
Gerrit-PatchSet: 9
Gerrit-Owner: Xqt <info@gno.de>
Gerrit-Reviewer: D3r1ck01 <dalangi-ctr@wikimedia.org>
Gerrit-Reviewer: JJMC89 <JJMC89.Wikimedia@gmail.com>
Gerrit-Reviewer: Multichill <maarten@mdammers.nl>
Gerrit-Reviewer: jenkins-bot
Gerrit-CC: JAn Dudík <jan.dudik@gmail.com>
Gerrit-CC: Mpaa <mpaa.wiki@gmail.com>
Gerrit-MessageType: merged