jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/598269 )

Change subject: [IMPR] Throttle requests after ratelimits exceeded ......................................................................

[IMPR] Throttle requests after ratelimits exceeded

- add 'ratelimits' to uiprop to get the rate limits - Handle 'ratelimited' response code: * check for an appropriate rate limit for the current action * calculate the delay for the next try * wait that time or use the default wait cycle - modify the Request.wait method to respect a delay as parameter

Bug: T253180 Change-Id: I28ca14e7b9d8c8296b978ac506955e15f8728090 --- M pywikibot/data/api.py M pywikibot/site/__init__.py 2 files changed, 27 insertions(+), 5 deletions(-)

Approvals: JJMC89: Looks good to me, approved jenkins-bot: Verified

diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index cca8864..243dd7f 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -1912,6 +1912,23 @@ self.wait() return True

+ def _ratelimited(self): + """Handle ratelimited warning.""" + ratelimits = self.site.userinfo['ratelimits'] + delay = None + + ratelimit = ratelimits.get(self.action, {}) + # find the lowest wait time for the given action + for limit in ratelimit.values(): + seconds = limit['seconds'] + hits = limit['hits'] + delay = min(delay or seconds, seconds / hits) + + if not delay: + pywikibot.warning( + 'No rate limit found for action {}'.format(self.action)) + self.wait(delay) + def _bad_token(self, code): """Check for bad token.""" if (code != 'badtoken' or self.site._loginstatus @@ -2047,6 +2064,10 @@ self.wait() continue

+ if code == 'ratelimited': + self._ratelimited() + continue + # If readapidenied is returned try to login if code == 'readapidenied' and self.site._loginstatus in (-3, -1): self.site.login() @@ -2100,14 +2121,15 @@

raise MaxlagTimeoutError(msg)

- def wait(self): + def wait(self, delay=None): """Determine how long to wait after a failed request.""" self.max_retries -= 1 if self.max_retries < 0: raise TimeoutError('Maximum retries attempted without success.') - pywikibot.warning('Waiting %s seconds before retrying.' - % self.retry_wait) - pywikibot.sleep(self.retry_wait) + delay = delay or self.retry_wait + pywikibot.warning('Waiting {:.1g} seconds before retrying.' + .format(delay)) + pywikibot.sleep(delay) # double the next wait, but do not exceed config.retry_max seconds self.retry_wait = min(config.retry_max, self.retry_wait * 2)

diff --git a/pywikibot/site/__init__.py b/pywikibot/site/__init__.py index 7dcff1c..7e3b5b7 100644 --- a/pywikibot/site/__init__.py +++ b/pywikibot/site/__init__.py @@ -2159,7 +2159,7 @@ uirequest = self._simple_request( action='query', meta='userinfo', - uiprop='blockinfo|hasmsg|groups|rights' + uiprop='blockinfo|hasmsg|groups|rights|ratelimits' ) uidata = uirequest.submit() assert 'query' in uidata, \