jenkins-bot merged this change.

View Change

[IMPR] Throttle requests after ratelimits exceeded

- add 'ratelimits' to uiprop to get the rate limits
- Handle 'ratelimited' response code:
  * check for an appropriate rate limit for the current action
  * calculate the delay for the next try
  * wait that time or use the default wait cycle
- modify the Request.wait method to respect a delay as parameter

Bug: T253180
Change-Id: I28ca14e7b9d8c8296b978ac506955e15f8728090
---
M pywikibot/data/api.py
M pywikibot/site/__init__.py
2 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index cca8864..243dd7f 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -1912,6 +1912,23 @@
         self.wait()
         return True
 
+    def _ratelimited(self):
+        """Handle ratelimited warning."""
+        ratelimits = self.site.userinfo['ratelimits']
+        delay = None
+
+        ratelimit = ratelimits.get(self.action, {})
+        # find the lowest wait time for the given action
+        for limit in ratelimit.values():
+            seconds = limit['seconds']
+            hits = limit['hits']
+            delay = min(delay or seconds, seconds / hits)
+
+        if not delay:
+            pywikibot.warning(
+                'No rate limit found for action {}'.format(self.action))
+        self.wait(delay)
+
     def _bad_token(self, code):
         """Check for bad token."""
         if (code != 'badtoken' or self.site._loginstatus
@@ -2047,6 +2064,10 @@
                 self.wait()
                 continue
 
+            if code == 'ratelimited':
+                self._ratelimited()
+                continue
+
             # If readapidenied is returned try to login
             if code == 'readapidenied' and self.site._loginstatus in (-3, -1):
                 self.site.login()
@@ -2100,14 +2121,15 @@
 
         raise MaxlagTimeoutError(msg)
 
-    def wait(self):
+    def wait(self, delay=None):
         """Determine how long to wait after a failed request."""
         self.max_retries -= 1
         if self.max_retries < 0:
             raise TimeoutError('Maximum retries attempted without success.')
-        pywikibot.warning('Waiting %s seconds before retrying.'
-                          % self.retry_wait)
-        pywikibot.sleep(self.retry_wait)
+        delay = delay or self.retry_wait
+        pywikibot.warning('Waiting {:.1g} seconds before retrying.'
+                          .format(delay))
+        pywikibot.sleep(delay)
         # double the next wait, but do not exceed config.retry_max seconds
         self.retry_wait = min(config.retry_max, self.retry_wait * 2)
 
diff --git a/pywikibot/site/__init__.py b/pywikibot/site/__init__.py
index 7dcff1c..7e3b5b7 100644
--- a/pywikibot/site/__init__.py
+++ b/pywikibot/site/__init__.py
@@ -2159,7 +2159,7 @@
             uirequest = self._simple_request(
                 action='query',
                 meta='userinfo',
-                uiprop='blockinfo|hasmsg|groups|rights'
+                uiprop='blockinfo|hasmsg|groups|rights|ratelimits'
             )
             uidata = uirequest.submit()
             assert 'query' in uidata, \

To view, visit change 598269. To unsubscribe, or for help writing mail filters, visit settings.