jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/949003 )
Change subject: [IMPR] Add a WaitingMixin to Requests and SparqlQuery class ......................................................................
[IMPR] Add a WaitingMixin to Requests and SparqlQuery class
Add a WaitingMixin class which handles wait cycles to Requests and SparqlQuery class to prevent code duplication.
Change-Id: Id1f528044a4d95587431b0ccd7fa10330c6a9826 --- M pywikibot/data/sparql.py M pywikibot/data/__init__.py M pywikibot/data/api/_requests.py 3 files changed, 81 insertions(+), 45 deletions(-)
Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified
diff --git a/pywikibot/data/__init__.py b/pywikibot/data/__init__.py index 7db53e3..1dd0b02 100644 --- a/pywikibot/data/__init__.py +++ b/pywikibot/data/__init__.py @@ -1,6 +1,56 @@ """Module providing several layers of data access to the wiki.""" # -# (C) Pywikibot team, 2007-2022 +# (C) Pywikibot team, 2007-2023 # # Distributed under the terms of the MIT license. # +from typing import Optional + +import pywikibot + + +class WaitingMixin: + + """A mixin to implement wait cycles. + + .. versionadded:: 8.4 + + :ivar int max_retries: Maximum number of times to retry an API + request before quitting. Defaults to ``config.max_retries`` if + attribute is missing. + :ivar int retry_wait: Minimum time to wait before resubmitting a + failed API request. Defaults to ``config.retry_wait`` if + attribute is missing. + :ivar int current_retries: counter of retries made for the current + request. Starting with 1 if attribute is missing. + """ + + def wait(self, delay: Optional[int] = None) -> None: + """Determine how long to wait after a failed request. + + :param delay: Minimum time in seconds to wait. Overwrites + ``retry_wait`` variable if given. The delay doubles each + retry until ``retry_max`` seconds is reached. + """ + if not hasattr(self, 'max_retries'): + self.max_retries = pywikibot.config.max_retries + + if not hasattr(self, 'retry_wait'): + self.retry_wait = pywikibot.config.retry_wait + + if self.current_retries > self.max_retries: + raise pywikibot.exceptions.TimeoutError( + 'Maximum retries attempted without success.') + + if not hasattr(self, 'current_retries'): + self.current_retries = 1 + else: + self.current_retries += 1 + + # double the next wait, but do not exceed config.retry_max seconds + delay = delay or self.retry_wait + delay *= 2 ** (self.current_retries - 1) + delay = min(delay, pywikibot.config.retry_max) + + pywikibot.warning(f'Waiting {delay:.1f} seconds before retrying.') + pywikibot.sleep(delay) diff --git a/pywikibot/data/api/_requests.py b/pywikibot/data/api/_requests.py index 7e7758c..e13bd6f 100644 --- a/pywikibot/data/api/_requests.py +++ b/pywikibot/data/api/_requests.py @@ -24,6 +24,7 @@ from pywikibot import config from pywikibot.backports import Callable, Dict, Match, Tuple, removeprefix from pywikibot.comms import http +from pywikibot.data import WaitingMixin from pywikibot.exceptions import ( Client414Error, Error, @@ -32,7 +33,6 @@ NoUsernameError, Server504Error, SiteDefinitionError, - TimeoutError, ) from pywikibot.login import LoginStatus from pywikibot.textlib import removeDisabledParts, removeHTMLParts @@ -72,7 +72,7 @@ r'Waiting for [\w.: ]+: (?P<lag>\d+(?:.\d+)?) seconds? lagged')
-class Request(MutableMapping): +class Request(MutableMapping, WaitingMixin):
"""A request to a Site's api.php interface.
@@ -126,6 +126,9 @@ True >>> sorted(data['query']) ['namespaces', 'userinfo'] + + .. versionchanged:: 8.4 + inherited from WaitingMixin. """
# To make sure the default value of 'parameters' can be identified. @@ -194,14 +197,9 @@
self.throttle = throttle self.use_get = use_get - if max_retries is None: - self.max_retries = pywikibot.config.max_retries - else: + if max_retries is not None: self.max_retries = max_retries - self.current_retries = 0 - if retry_wait is None: - self.retry_wait = pywikibot.config.retry_wait - else: + if retry_wait is not None: self.retry_wait = retry_wait self.json_warning = False # The only problem with that system is that it won't detect when @@ -1120,20 +1118,6 @@
raise MaxlagTimeoutError(msg)
- def wait(self, delay=None): - """Determine how long to wait after a failed request.""" - self.current_retries += 1 - if self.current_retries > self.max_retries: - raise TimeoutError('Maximum retries attempted without success.') - - # double the next wait, but do not exceed config.retry_max seconds - delay = delay or self.retry_wait - delay *= 2 ** (self.current_retries - 1) - delay = min(delay, config.retry_max) - - pywikibot.warning(f'Waiting {delay:.1f} seconds before retrying.') - pywikibot.sleep(delay) -
class CachedRequest(Request):
diff --git a/pywikibot/data/sparql.py b/pywikibot/data/sparql.py index 5c4e14e..23ffb1d 100644 --- a/pywikibot/data/sparql.py +++ b/pywikibot/data/sparql.py @@ -10,10 +10,11 @@
from requests.exceptions import Timeout
-from pywikibot import Site, config, sleep, warning +from pywikibot import Site from pywikibot.backports import Dict, List, removeprefix from pywikibot.comms import http -from pywikibot.exceptions import Error, TimeoutError +from pywikibot.data import WaitingMixin +from pywikibot.exceptions import Error
try: @@ -25,11 +26,14 @@ 'Accept': 'application/sparql-results+json'}
-class SparqlQuery: - """ - SPARQL Query class. +class SparqlQuery(WaitingMixin): + """SPARQL Query class.
This class allows to run SPARQL queries against any SPARQL endpoint. + + .. versionchanged:: 8.4 + inherited from :class:`data.WaitingMixin` which provides a + :meth:`data.WaitingMixin.wait` method. """
def __init__(self, @@ -78,13 +82,9 @@
self.last_response = None
- if max_retries is None: - self.max_retries = config.max_retries - else: + if max_retries is not None: self.max_retries = max_retries - if retry_wait is None: - self.retry_wait = config.retry_wait - else: + if retry_wait is not None: self.retry_wait = retry_wait
def get_last_response(self): @@ -158,16 +158,6 @@
return None
- def wait(self): - """Determine how long to wait after a failed request.""" - self.max_retries -= 1 - if self.max_retries < 0: - raise TimeoutError('Maximum retries attempted without success.') - warning(f'Waiting {self.retry_wait} seconds before retrying.') - sleep(self.retry_wait) - # double the next wait, but do not exceed config.retry_max seconds - self.retry_wait = min(config.retry_max, self.retry_wait * 2) - def ask(self, query: str, headers: Optional[Dict[str, str]] = None) -> bool: """