jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/351758 )
Change subject: weblinkchecker.py: Retry get_memento_info if there is a ConnectionError ......................................................................
weblinkchecker.py: Retry get_memento_info if there is a ConnectionError
`get_memento_info` sometimes fails to connect to the server. Retrying should mitigate the issue.
Skip the related test on ConnectionError. `max_retries` is reduced to 1 during tests and thus the test might still fail.
Bug: T164365 Change-Id: I0db18b7a3e5fa67de4132f73c58a15b76725edc3 --- M scripts/weblinkchecker.py M tests/weblinkchecker_tests.py 2 files changed, 23 insertions(+), 6 deletions(-)
Approvals: jenkins-bot: Verified Xqt: Looks good to me, approved
diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py index 9fd0274..e261164 100755 --- a/scripts/weblinkchecker.py +++ b/scripts/weblinkchecker.py @@ -113,6 +113,7 @@ import time
from functools import partial +from time import sleep from warnings import warn
try: @@ -122,7 +123,9 @@
import pywikibot
-from pywikibot import comms, i18n, config, pagegenerators, textlib, weblib +from pywikibot import ( + comms, i18n, config, pagegenerators, textlib, weblib, config2, +)
from pywikibot.bot import ExistingPageBot, SingleSiteBot from pywikibot.pagegenerators import ( @@ -188,7 +191,18 @@ if timegate_uri: mc.timegate_uri = timegate_uri
- memento_info = mc.get_memento_info(url, when) + retry_count = 0 + while retry_count <= config2.max_retries: + try: + memento_info = mc.get_memento_info(url, when) + break + except requests.ConnectionError as e: + error = e + retry_count += 1 + sleep(config2.retry_wait) + else: + raise error + mementos = memento_info.get('mementos') if not mementos: raise Exception( diff --git a/tests/weblinkchecker_tests.py b/tests/weblinkchecker_tests.py index be84bab..63171ac 100644 --- a/tests/weblinkchecker_tests.py +++ b/tests/weblinkchecker_tests.py @@ -9,6 +9,8 @@
import datetime
+from requests import ConnectionError as RequestsConnectionError + from pywikibot.tools import PY2 if not PY2: from urllib.parse import urlparse @@ -31,10 +33,11 @@ when = datetime.datetime.now() else: when = datetime.datetime.strptime(date_string, '%Y%m%d') - return weblinkchecker._get_closest_memento_url( - url, - when, - self.timegate_uri) + try: + return weblinkchecker._get_closest_memento_url( + url, when, self.timegate_uri) + except RequestsConnectionError as e: + self.skipTest(e)
class WeblibTestMementoInternetArchive(MementoTestCase, weblib_tests.TestInternetArchive):