jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/678395 )
Change subject: [IMPR] use http.HTTPStatus description, phrase and value ......................................................................
[IMPR] use http.HTTPStatus description, phrase and value
Change-Id: Iee4c6f20bbf3f62f4175016076768e9a176b1b50 --- M pywikibot/comms/http.py M pywikibot/page/__init__.py M pywikibot/pagegenerators.py M pywikibot/proofreadpage.py M pywikibot/site_detect.py M pywikibot/specialbots/_upload.py M scripts/download_dump.py M scripts/reflinks.py M scripts/weblinkchecker.py M tests/aspects.py M tests/http_tests.py M tests/site_detect_tests.py M tests/site_tests.py 13 files changed, 79 insertions(+), 49 deletions(-)
Approvals: DannyS712: Looks good to me, but someone else must approve Xqt: Looks good to me, approved jenkins-bot: Verified
diff --git a/pywikibot/comms/http.py b/pywikibot/comms/http.py index 5807013..65cddd1 100644 --- a/pywikibot/comms/http.py +++ b/pywikibot/comms/http.py @@ -21,7 +21,7 @@ import sys
from contextlib import suppress -from http import cookiejar +from http import cookiejar, HTTPStatus from string import Formatter from typing import Optional, Union from urllib.parse import quote, urlparse @@ -295,11 +295,11 @@ error('An error occurred for uri ' + response.request.url) raise response from None
- if response.status_code == 504: + if response.status_code == HTTPStatus.GATEWAY_TIMEOUT: raise Server504Error('Server {} timed out' .format(urlparse(response.url).netloc))
- if response.status_code == 414: + if response.status_code == HTTPStatus.REQUEST_URI_TOO_LONG: raise Server414Error('Too long GET request')
# TODO: shall it raise? this might break some code, TBC @@ -307,7 +307,7 @@
# HTTP status 207 is also a success status for Webdav FINDPROP, # used by the version module. - if response.status_code not in (200, 207): + if response.status_code not in (HTTPStatus.OK, HTTPStatus.MULTI_STATUS): warning('Http response status {}'.format(response.status_code))
diff --git a/pywikibot/page/__init__.py b/pywikibot/page/__init__.py index 17a2814..a5aa643 100644 --- a/pywikibot/page/__init__.py +++ b/pywikibot/page/__init__.py @@ -23,6 +23,7 @@ from collections import Counter, defaultdict, OrderedDict from contextlib import suppress from html.entities import name2codepoint +from http import HTTPStatus from itertools import chain from typing import Any, Optional, Union from urllib.parse import quote_from_bytes, unquote_to_bytes @@ -2458,7 +2459,7 @@ revision = self.latest_file_info
req = http.fetch(revision.url, stream=True) - if req.status_code == 200: + if req.status_code == HTTPStatus.OK: try: with open(filename, 'wb') as f: for chunk in req.iter_content(chunk_size): diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index fbc585f..ce0398d 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -29,6 +29,7 @@ from collections import namedtuple from datetime import timedelta from functools import partial +from http import HTTPStatus from itertools import zip_longest from requests.exceptions import ReadTimeout from typing import Optional, Union @@ -2967,11 +2968,13 @@ try: req = http.fetch(url, params=self.opts) except ReadTimeout: - raise ServerError( - 'received ReadTimeout from {}'.format(url)) - if 500 <= req.status_code < 600: + raise ServerError('received ReadTimeout from {}'.format(url)) + + server_err = HTTPStatus.INTERNAL_SERVER_ERROR + if server_err <= req.status_code < server_err + 100: raise ServerError( 'received {} status from {}'.format(req.status_code, req.url)) + j = json.loads(req.text) raw_pages = j['*'][0]['a']['*'] yield from raw_pages diff --git a/pywikibot/proofreadpage.py b/pywikibot/proofreadpage.py index 92ec48e..e8cb902 100644 --- a/pywikibot/proofreadpage.py +++ b/pywikibot/proofreadpage.py @@ -28,9 +28,11 @@ import time
from functools import partial -from requests.exceptions import ReadTimeout +from http import HTTPStatus from typing import Optional
+from requests.exceptions import ReadTimeout + try: from bs4 import BeautifulSoup except ImportError as e: @@ -618,7 +620,7 @@ else: return True, ReadTimeout
- if 400 <= response.status_code < 600: + if HTTPStatus.BAD_REQUEST <= response.status_code < 600: return True, 'Http response status {}'.format(response.status_code)
data = json.loads(response.text) diff --git a/pywikibot/site_detect.py b/pywikibot/site_detect.py index b509fb0..dcc6117 100644 --- a/pywikibot/site_detect.py +++ b/pywikibot/site_detect.py @@ -8,8 +8,10 @@
from contextlib import suppress from html.parser import HTMLParser +from http import HTTPStatus from typing import Optional from urllib.parse import urljoin, urlparse + from requests.exceptions import RequestException
import pywikibot @@ -273,11 +275,9 @@
def check_response(response): """Raise ServerError if the response indicates a server error.""" - if response.status_code == 503: - raise ServerError('Service Unavailable') - if response.status_code == 502: - raise ServerError('Bad Gateway') - if response.status_code == 500: - raise ServerError('Internal Server Error') - if response.status_code == 200 and SERVER_DB_ERROR_MSG in response.text: + if response.status_code >= HTTPStatus.INTERNAL_SERVER_ERROR: + raise ServerError(HTTPStatus(response.status_code).phrase) + + if response.status_code == HTTPStatus.OK \ + and SERVER_DB_ERROR_MSG in response.text: raise ServerError('Server cannot access the database') diff --git a/pywikibot/specialbots/_upload.py b/pywikibot/specialbots/_upload.py index 5f76d8f..8b1f49a 100644 --- a/pywikibot/specialbots/_upload.py +++ b/pywikibot/specialbots/_upload.py @@ -13,6 +13,7 @@ import tempfile
from contextlib import suppress +from http import HTTPStatus from pathlib import Path from typing import Optional, Union from urllib.parse import urlparse @@ -158,7 +159,8 @@ # exit criteria if size is not available # error on last iteration is OK, we're requesting # {'Range': 'bytes=file_len-'} - if response.status_code == 416 and path.stat().st_size: + err = HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE + if response.status_code == err and path.stat().st_size: break raise FatalServerError(str(e)) from e
diff --git a/scripts/download_dump.py b/scripts/download_dump.py index 4f38605..9ba08e7 100755 --- a/scripts/download_dump.py +++ b/scripts/download_dump.py @@ -20,6 +20,7 @@ import binascii import os.path
+from http import HTTPStatus from os import remove, replace, symlink, urandom
import pywikibot @@ -108,13 +109,16 @@ pywikibot.output('Downloading file from ' + url) response = fetch(url, stream=True)
- if response.status_code != 200: - if response.status_code == 404: + if response.status_code != HTTPStatus.OK: + if response.status_code == HTTPStatus.NOT_FOUND: pywikibot.output( 'File with name {filename!r}, from dumpdate ' '{dumpdate!r}, and wiki {wikiname!r} ({url}) ' "isn't available in the Wikimedia Dumps" .format(url=url, **self.opt)) + else: + pywikibot.output( + HTTPStatus(response.status_code).description) return
with open(file_current_storepath, 'wb') as result_file: diff --git a/scripts/reflinks.py b/scripts/reflinks.py index 559da2e..9c816f4 100755 --- a/scripts/reflinks.py +++ b/scripts/reflinks.py @@ -49,11 +49,10 @@
from contextlib import suppress from functools import partial +from http import HTTPStatus from textwrap import shorten from urllib.error import URLError
-from requests import codes - import pywikibot
from pywikibot import comms, i18n, pagegenerators, textlib @@ -584,16 +583,15 @@ 'Redirect to root : {0} ', ref.link)) continue
- if r.status_code != codes.ok: + if r.status_code != HTTPStatus.OK: pywikibot.stdout('HTTP error ({}) for {} on {}' .format(r.status_code, ref.url, page.title(as_link=True))) # 410 Gone, indicates that the resource has been # purposely removed - if r.status_code == 410 \ - or (r.status_code == 404 - and '\t{}\t'.format( - ref.url) in self.dead_links): + if r.status_code == HTTPStatus.GONE \ + or (r.status_code == HTTPStatus.NOT_FOUND + and '\t{}\t'.format(ref.url) in self.dead_links): repl = ref.refDead() new_text = new_text.replace(match.group(), repl) continue diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py index 192fc9f..b2b42cb 100755 --- a/scripts/weblinkchecker.py +++ b/scripts/weblinkchecker.py @@ -117,6 +117,7 @@
from contextlib import suppress from functools import partial +from http import HTTPStatus
import requests
@@ -321,9 +322,8 @@ .format(self.url, self.page.title())) raise
- if r.status_code != requests.codes.ok \ - or r.status_code in self.HTTPignore: - message = str(r.status_code) + if r.status_code != HTTPStatus.OK or r.status_code in self.HTTPignore: + message = HTTPStatus(r.status_code).phrase pywikibot.output('*{} links to {} - {}.' .format(self.page.title(as_link=True), self.url, message)) diff --git a/tests/aspects.py b/tests/aspects.py index 18caee2..87da3bb 100644 --- a/tests/aspects.py +++ b/tests/aspects.py @@ -20,6 +20,7 @@
from contextlib import contextmanager, suppress from collections.abc import Sized +from http import HTTPStatus from unittest.util import safe_repr
import pywikibot @@ -451,8 +452,15 @@ r = http.fetch(hostname, method='HEAD', default_error_handling=False) - if r.status_code not in {200, 301, 302, 303, 307, 308}: - raise ServerError('HTTP status: {}'.format(r.status_code)) + if r.status_code not in {HTTPStatus.OK, + HTTPStatus.MOVED_PERMANENTLY, + HTTPStatus.FOUND, + HTTPStatus.SEE_OTHER, + HTTPStatus.TEMPORARY_REDIRECT, + HTTPStatus.PERMANENT_REDIRECT}: + raise ServerError( + 'HTTP status: {} - {}'.format( + r.status_code, HTTPStatus(r.status_code).phrase)) except Exception as e: pywikibot.error('{}: accessing {} caused exception:' .format(cls.__name__, hostname)) diff --git a/tests/http_tests.py b/tests/http_tests.py index aa2fa67..5656c5d 100644 --- a/tests/http_tests.py +++ b/tests/http_tests.py @@ -9,6 +9,7 @@ import warnings
from contextlib import suppress +from http import HTTPStatus
import requests
@@ -42,7 +43,7 @@ """Test http.fetch using http://www.wikipedia.org/.""" r = http.fetch('http://www.wikipedia.org/') self.assertIsInstance(r, requests.Response) - self.assertEqual(r.status_code, 200) + self.assertEqual(r.status_code, HTTPStatus.OK.value) self.assertIn('<html lang="en"', r.text) self.assertIsInstance(r.text, str) self.assertIsInstance(r.content, bytes) @@ -167,12 +168,12 @@ """Test follow 301 redirects correctly.""" # The following will redirect from ' ' -> '_', and maybe to https:// r = http.fetch('http://en.wikipedia.org/wiki/Main%20Page') - self.assertEqual(r.status_code, 200) + self.assertEqual(r.status_code, HTTPStatus.OK.value) self.assertIsNotNone(r.history) self.assertIn('//en.wikipedia.org/wiki/Main_Page', r.url)
r = http.fetch('http://en.wikia.com') - self.assertEqual(r.status_code, 200) + self.assertEqual(r.status_code, HTTPStatus.OK.value) self.assertEqual(r.url, 'https://community.fandom.com/wiki/Community_Central')
@@ -555,10 +556,13 @@ def test_no_params(self): """Test fetch method with no parameters.""" r = http.fetch(self.url, params={}) - if r.status_code == 503: # T203637 - self.skipTest( - '503: Service currently not available for ' + self.url) - self.assertEqual(r.status_code, 200) + + fail_status = HTTPStatus.SERVICE_UNAVAILABLE + if r.status_code == fail_status: # T203637 + self.skipTest('{status.value}: {status.description} for {url}' + .format(status=fail_status, url=self.url)) + + self.assertEqual(r.status_code, HTTPStatus.OK)
content = json.loads(r.text) self.assertDictEqual(content['args'], {}) @@ -571,10 +575,13 @@ should be the same as what we get out. """ r = http.fetch(self.url, params={'fish&chips': 'delicious'}) - if r.status_code == 503: # T203637 - self.skipTest( - '503: Service currently not available for ' + self.url) - self.assertEqual(r.status_code, 200) + + fail_status = HTTPStatus.SERVICE_UNAVAILABLE + if r.status_code == fail_status: # T203637 + self.skipTest('{status.value}: {status.description} for {url}' + .format(status=fail_status, url=self.url)) + + self.assertEqual(r.status_code, HTTPStatus.OK)
content = json.loads(r.text) self.assertDictEqual(content['args'], {'fish&chips': 'delicious'}) @@ -587,10 +594,13 @@ should be the same as what we get out. """ r = http.fetch(self.url, params={'fish%26chips': 'delicious'}) - if r.status_code == 503: # T203637 - self.skipTest( - '503: Service currently not available for ' + self.url) - self.assertEqual(r.status_code, 200) + + fail_status = HTTPStatus.SERVICE_UNAVAILABLE + if r.status_code == fail_status: # T203637 + self.skipTest('{status.value}: {status.description} for {url}' + .format(status=fail_status, url=self.url)) + + self.assertEqual(r.status_code, HTTPStatus.OK)
content = json.loads(r.text) self.assertDictEqual(content['args'], {'fish%26chips': 'delicious'}) diff --git a/tests/site_detect_tests.py b/tests/site_detect_tests.py index 813e890..807d105 100644 --- a/tests/site_detect_tests.py +++ b/tests/site_detect_tests.py @@ -7,6 +7,7 @@ import unittest
from contextlib import suppress +from http import HTTPStatus from urllib.parse import urlparse
from requests.exceptions import ConnectionError, Timeout @@ -290,7 +291,7 @@
return type(str('Response'), (object,), - {'status_code': 200, + {'status_code': HTTPStatus.OK.value, 'text': self._responses[parsed_url.path], 'url': url})
diff --git a/tests/site_tests.py b/tests/site_tests.py index 677d1a4..877445d 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -12,6 +12,7 @@
from collections.abc import Iterable, Mapping from contextlib import suppress +from http import HTTPStatus
import pywikibot
@@ -3287,7 +3288,7 @@ self.assertEqual(site.hostname(), 'mh.wikipedia.org') r = http.fetch('http://mh.wikipedia.org/w/api.php', default_error_handling=False) - self.assertEqual(r.status_code, 200) + self.assertEqual(r.status_code, HTTPStatus.OK.value) self.assertEqual(site.siteinfo['lang'], 'mh')
def test_removed_site(self):