jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/421282 ) Change subject: threadedhttp.py: Deprecate HttpRequest.content in favour of text ...................................................................... threadedhttp.py: Deprecate HttpRequest.content in favour of text Bug: T173352 Change-Id: I2769f207fd297969c2de38996a6c741fd9f597a0 --- M pywikibot/comms/http.py M pywikibot/comms/threadedhttp.py M pywikibot/data/sparql.py M pywikibot/pagegenerators.py M pywikibot/proofreadpage.py M pywikibot/site_detect.py M pywikibot/version.py M pywikibot/weblib.py M scripts/flickrripper.py M scripts/imagerecat.py M scripts/reflinks.py M tests/http_tests.py M tests/sparql_tests.py M tests/weblib_tests.py 14 files changed, 55 insertions(+), 46 deletions(-) Approvals: Mpaa: Looks good to me, but someone else must approve Framawiki: Looks good to me, but someone else must approve Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/comms/http.py b/pywikibot/comms/http.py index 76878ae..d98ab8a 100644 --- a/pywikibot/comms/http.py +++ b/pywikibot/comms/http.py @@ -14,7 +14,7 @@ from __future__ import absolute_import, print_function, unicode_literals # -# (C) Pywikibot team, 2007-2017 +# (C) Pywikibot team, 2007-2018 # # Distributed under the terms of the MIT license. # @@ -303,7 +303,7 @@ issue_deprecation_warning( 'Invoking http.request without argument site', 'http.fetch()', 3) r = fetch(uri, method, params, body, headers, **kwargs) - return r.content + return r.text baseuri = site.base_url(uri) @@ -319,7 +319,7 @@ headers['user-agent'] = user_agent(site, format_string) r = fetch(baseuri, method, params, body, headers, **kwargs) - return r.content + return r.text def get_authentication(uri): diff --git a/pywikibot/comms/threadedhttp.py b/pywikibot/comms/threadedhttp.py index 835fa50..4eadb57 100644 --- a/pywikibot/comms/threadedhttp.py +++ b/pywikibot/comms/threadedhttp.py @@ -3,7 +3,7 @@ from __future__ import absolute_import, unicode_literals # -# (C) Pywikibot team, 2007-2017 +# (C) Pywikibot team, 2007-2018 # # Distributed under the terms of the MIT license. # @@ -22,7 +22,7 @@ import pywikibot -from pywikibot.tools import UnicodeMixin +from pywikibot.tools import deprecated, UnicodeMixin _logger = "comm.threadedhttp" @@ -189,13 +189,19 @@ return self.raw.decode(encoding, errors) @property + @deprecated('the `text` property') def content(self): + """DEPRECATED. Return the response decoded by the detected encoding.""" + return self.text + + @property + def text(self): """Return the response decoded by the detected encoding.""" return self.decode(self.encoding) def __unicode__(self): """Return the response decoded by the detected encoding.""" - return self.content + return self.text def __bytes__(self): """Return the undecoded response.""" diff --git a/pywikibot/data/sparql.py b/pywikibot/data/sparql.py index 392f8a8..13ec4ca 100644 --- a/pywikibot/data/sparql.py +++ b/pywikibot/data/sparql.py @@ -144,10 +144,10 @@ while True: try: self.last_response = http.fetch(url, headers=headers) - if not self.last_response.content: + if not self.last_response.text: return None try: - return json.loads(self.last_response.content) + return json.loads(self.last_response.text) except ValueError: return None except Timeout: diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index 55a5c1b..0392e0f 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -2961,7 +2961,7 @@ if 500 <= req.status < 600: raise ServerError( 'received {0} status from {1}'.format(req.status, req.uri)) - j = json.loads(req.content) + j = json.loads(req.text) raw_pages = j['*'][0]['a']['*'] for raw_page in raw_pages: yield raw_page diff --git a/pywikibot/proofreadpage.py b/pywikibot/proofreadpage.py index 5b36e5b..44059f4 100644 --- a/pywikibot/proofreadpage.py +++ b/pywikibot/proofreadpage.py @@ -18,7 +18,7 @@ """ # -# (C) Pywikibot team, 2015-2017 +# (C) Pywikibot team, 2015-2018 # # Distributed under the terms of the MIT license. # @@ -527,7 +527,7 @@ pywikibot.error('Error fetching HTML for %s.' % self) raise - soup = Soup(response.content) + soup = Soup(response.text) try: self._url_image = soup.find(class_='prp-page-image') @@ -566,7 +566,7 @@ pywikibot.error('Querying %s: %s' % (cmd_uri, e)) return (True, e) - data = json.loads(response.content) + data = json.loads(response.text) assert 'error' in data, 'Error from phe-tools: %s' % data assert data['error'] in [0, 1], 'Error from phe-tools: %s' % data diff --git a/pywikibot/site_detect.py b/pywikibot/site_detect.py index 1d82dea..a9dea68 100644 --- a/pywikibot/site_detect.py +++ b/pywikibot/site_detect.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Classes for detecting a MediaWiki site.""" # -# (C) Pywikibot team, 2010-2015 +# (C) Pywikibot team, 2010-2018 # # Distributed under the terms of the MIT license. # @@ -63,7 +63,7 @@ self.fromurl = fromurl - data = r.content + data = r.text wp = WikiHTMLPageParser(fromurl) wp.feed(data) @@ -118,7 +118,7 @@ response = fetch( self.api + "?action=query&meta=siteinfo&siprop=interwikimap&sifilteriw=local&format=json") - iw = json.loads(response.content) + iw = json.loads(response.text) if 'error' in iw: raise RuntimeError('%s - %s' % (iw['error']['code'], iw['error']['info'])) @@ -146,7 +146,7 @@ """Extract the version from API help with ?version enabled.""" if self.version is None: try: - d = fetch(self.api + '?version&format=json').content + d = fetch(self.api + '?version&format=json').text try: d = json.loads(d) except ValueError: @@ -167,7 +167,7 @@ response = fetch(self.api + '?action=query&meta=siteinfo&format=json') check_response(response) # remove preleading newlines and Byte Order Mark (BOM), see T128992 - content = response.content.strip().lstrip('\uFEFF') + content = response.text.strip().lstrip('\uFEFF') info = json.loads(content) self.private_wiki = ('error' in info and info['error']['code'] == 'readapidenied') @@ -306,5 +306,5 @@ raise ServerError('Bad Gateway') elif response.status == 500: raise ServerError('Internal Server Error') - elif response.status == 200 and SERVER_DB_ERROR_MSG in response.content: + elif response.status == 200 and SERVER_DB_ERROR_MSG in response.text: raise ServerError('Server cannot access the database') diff --git a/pywikibot/version.py b/pywikibot/version.py index e671c90..77f4c34 100644 --- a/pywikibot/version.py +++ b/pywikibot/version.py @@ -383,7 +383,7 @@ url = 'https://gerrit.wikimedia.org/r/projects/pywikibot%2Fcore/branches/master' # Gerrit API responses include )]}' at the beginning, make sure to strip it out buf = http.fetch(uri=url, - headers={'user-agent': '{pwb}'}).content[4:] + headers={'user-agent': '{pwb}'}).text[4:] try: hsh = json.loads(buf)['revision'] diff --git a/pywikibot/weblib.py b/pywikibot/weblib.py index 3cf13c6..87494fa 100644 --- a/pywikibot/weblib.py +++ b/pywikibot/weblib.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Functions for manipulating external links or querying third-party sites.""" # -# (C) Pywikibot team, 2013 +# (C) Pywikibot team, 2013-2018 # # Distributed under the terms of the MIT license. # @@ -48,7 +48,7 @@ retry_count = 0 while retry_count <= config2.max_retries: try: - jsontext = http.fetch(uri).content + jsontext = http.fetch(uri).text break except RequestsConnectionError as e: error = e @@ -85,7 +85,7 @@ query['date'] = timestamp uri = uri + urlencode(query) - xmltext = http.fetch(uri).content + xmltext = http.fetch(uri).text if "success" in xmltext: data = ET.fromstring(xmltext) return data.find('.//webcite_url').text diff --git a/scripts/flickrripper.py b/scripts/flickrripper.py index b03604d..3ef2fb2 100755 --- a/scripts/flickrripper.py +++ b/scripts/flickrripper.py @@ -168,7 +168,7 @@ parameters = urlencode({'id': photo_id, 'raw': 'on'}) return fetch( - 'http://wikipedia.ramselehof.de/flinfo.php?%s' % parameters).content + 'http://wikipedia.ramselehof.de/flinfo.php?%s' % parameters).text def getFilename(photoInfo, site=None, project=u'Flickr'): diff --git a/scripts/imagerecat.py b/scripts/imagerecat.py index ea4c5d2..884da46 100755 --- a/scripts/imagerecat.py +++ b/scripts/imagerecat.py @@ -166,7 +166,7 @@ commonsHelperPage = fetch( "https://toolserver.org/~daniel/WikiSense/CommonSense.php?%s" % parameters) matches = commonsenseRe.search( - commonsHelperPage.content) + commonsHelperPage.text) gotInfo = True else: break @@ -229,7 +229,7 @@ while not gotInfo: try: page = fetch('https://nominatim.openstreetmap.org/reverse?format=xml&%s… % parameters) - et = xml.etree.ElementTree.fromstring(page.content) + et = xml.etree.ElementTree.fromstring(page.text) gotInfo = True except IOError: pywikibot.output(u'Got an IOError, let\'s try again') diff --git a/scripts/reflinks.py b/scripts/reflinks.py index b4356bc..d713f9f 100755 --- a/scripts/reflinks.py +++ b/scripts/reflinks.py @@ -462,7 +462,7 @@ pywikibot.output(u'PDF file.') fd, infile = tempfile.mkstemp() urlobj = os.fdopen(fd, 'w+') - urlobj.write(f.content) + urlobj.write(f.text) try: pdfinfo_out = subprocess.Popen([r"pdfinfo", "/dev/stdin"], diff --git a/tests/http_tests.py b/tests/http_tests.py index f0829a7..e348f5a 100644 --- a/tests/http_tests.py +++ b/tests/http_tests.py @@ -48,8 +48,8 @@ r = http._enqueue('http://www.wikipedia.org/') self.assertIsInstance(r, threadedhttp.HttpRequest) self.assertEqual(r.status, 200) - self.assertIn('<html lang="mul"', r.content) - self.assertIsInstance(r.content, unicode) + self.assertIn('<html lang="mul"', r.text) + self.assertIsInstance(r.text, unicode) self.assertIsInstance(r.raw, bytes) def test_fetch(self): @@ -57,8 +57,10 @@ r = http.fetch('http://www.wikipedia.org/') self.assertIsInstance(r, threadedhttp.HttpRequest) self.assertEqual(r.status, 200) - self.assertIn('<html lang="mul"', r.content) - self.assertIsInstance(r.content, unicode) + self.assertIn('<html lang="mul"', r.text) + self.assertIsInstance(r.text, unicode) + with suppress_warnings(r'.*HttpRequest\.content is deprecated'): + self.assertEqual(r.content, r.text) self.assertIsInstance(r.raw, bytes) @@ -150,7 +152,7 @@ response = http.fetch( uri='https://testssl-expire-r2i2.disig.sk/index.en.html', disable_ssl_certificate_validation=True) - r = response.content + r = response.text self.assertIsInstance(r, unicode) self.assertTrue(re.search(r'<title>.*</title>', r)) http.session.close() # clear the connection @@ -454,7 +456,7 @@ self.assertIsNone(req.charset) self.assertEqual('latin1', req.encoding) self.assertEqual(req.raw, CharsetTestCase.LATIN1_BYTES) - self.assertEqual(req.content, CharsetTestCase.STR) + self.assertEqual(req.text, CharsetTestCase.STR) def test_no_charset(self): """Test decoding without explicit charset.""" @@ -466,7 +468,7 @@ self.assertIsNone(req.charset) self.assertEqual('latin1', req.encoding) self.assertEqual(req.raw, CharsetTestCase.LATIN1_BYTES) - self.assertEqual(req.content, CharsetTestCase.STR) + self.assertEqual(req.text, CharsetTestCase.STR) def test_content_type_application_json_without_charset(self): """Test decoding without explicit charset but JSON content.""" @@ -537,7 +539,7 @@ self.assertIsNone(req.charset) self.assertEqual('utf-8', req.encoding) self.assertEqual(req.raw, CharsetTestCase.UTF8_BYTES) - self.assertEqual(req.content, CharsetTestCase.STR) + self.assertEqual(req.text, CharsetTestCase.STR) def test_same_charset(self): """Test decoding with explicit and equal charsets.""" @@ -545,7 +547,7 @@ self.assertEqual('utf-8', req.charset) self.assertEqual('utf-8', req.encoding) self.assertEqual(req.raw, CharsetTestCase.UTF8_BYTES) - self.assertEqual(req.content, CharsetTestCase.STR) + self.assertEqual(req.text, CharsetTestCase.STR) def test_header_charset(self): """Test decoding with different charsets and valid header charset.""" @@ -555,7 +557,7 @@ with patch('pywikibot.warning'): self.assertEqual('utf-8', req.encoding) self.assertEqual(req.raw, CharsetTestCase.UTF8_BYTES) - self.assertEqual(req.content, CharsetTestCase.STR) + self.assertEqual(req.text, CharsetTestCase.STR) def test_code_charset(self): """Test decoding with different charsets and invalid header charset.""" @@ -566,7 +568,7 @@ with patch('pywikibot.warning'): self.assertEqual('latin1', req.encoding) self.assertEqual(req.raw, CharsetTestCase.LATIN1_BYTES) - self.assertEqual(req.content, CharsetTestCase.STR) + self.assertEqual(req.text, CharsetTestCase.STR) def test_invalid_charset(self): """Test decoding with different and invalid charsets.""" @@ -579,7 +581,8 @@ UnicodeDecodeError, self.CODEC_CANT_DECODE_RE, lambda: req.encoding) self.assertEqual(req.raw, CharsetTestCase.LATIN1_BYTES) - self.assertRaisesRegex(UnicodeDecodeError, self.CODEC_CANT_DECODE_RE, lambda: req.content) + self.assertRaisesRegex( + UnicodeDecodeError, self.CODEC_CANT_DECODE_RE, lambda: req.text) class BinaryTestCase(TestCase): @@ -645,7 +648,7 @@ r = http.fetch(uri=self.get_httpbin_url('/get'), params={}) self.assertEqual(r.status, 200) - content = json.loads(r.content) + content = json.loads(r.text) self.assertDictEqual(content['args'], {}) def test_unencoded_params(self): @@ -658,7 +661,7 @@ r = http.fetch(uri=self.get_httpbin_url('/get'), params={'fish&chips': 'delicious'}) self.assertEqual(r.status, 200) - content = json.loads(r.content) + content = json.loads(r.text) self.assertDictEqual(content['args'], {'fish&chips': 'delicious'}) def test_encoded_params(self): @@ -672,7 +675,7 @@ params={'fish%26chips': 'delicious'}) self.assertEqual(r.status, 200) - content = json.loads(r.content) + content = json.loads(r.text) self.assertDictEqual(content['args'], {'fish%26chips': 'delicious'}) @@ -686,8 +689,8 @@ r_body = http.fetch(uri=self.get_httpbin_url('/post'), method='POST', body={'fish&chips': 'delicious'}) - self.assertDictEqual(json.loads(r_data.content), - json.loads(r_body.content)) + self.assertDictEqual(json.loads(r_data.text), + json.loads(r_body.text)) if __name__ == '__main__': # pragma: no cover diff --git a/tests/sparql_tests.py b/tests/sparql_tests.py index 933de95..838e567 100644 --- a/tests/sparql_tests.py +++ b/tests/sparql_tests.py @@ -87,7 +87,7 @@ def __init__(self, value): """Create container.""" - self.content = value + self.text = value class TestSparql(WikidataTestCase): diff --git a/tests/weblib_tests.py b/tests/weblib_tests.py index e4b5414..fe4982a 100644 --- a/tests/weblib_tests.py +++ b/tests/weblib_tests.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- """Weblib test module.""" # -# (C) Pywikibot team, 2014-2015 +# (C) Pywikibot team, 2014-2018 # # Distributed under the terms of the MIT license. # @@ -35,7 +35,7 @@ def _test_response(self, response, *args, **kwargs): # for later tests this is must be present, and it'll tell us the # original content if that does not match - self.assertIn('closest', response.content) + self.assertIn('closest', response.text) def _get_archive_url(self, url, date_string=None): with PatchedHttp(weblib, False) as p: -- To view, visit https://gerrit.wikimedia.org/r/421282 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-MessageType: merged Gerrit-Change-Id: I2769f207fd297969c2de38996a6c741fd9f597a0 Gerrit-Change-Number: 421282 Gerrit-PatchSet: 1 Gerrit-Owner: Dalba &lt;dalba.wiki(a)gmail.com&gt; Gerrit-Reviewer: Dalba &lt;dalba.wiki(a)gmail.com&gt; Gerrit-Reviewer: Framawiki &lt;framawiki(a)tools.wmflabs.org&gt; Gerrit-Reviewer: John Vandenberg &lt;jayvdb(a)gmail.com&gt; Gerrit-Reviewer: Mpaa &lt;mpaa.wiki(a)gmail.com&gt; Gerrit-Reviewer: Xqt &lt;info(a)gno.de&gt; Gerrit-Reviewer: Zoranzoki21 &lt;zorandori4444(a)gmail.com&gt; Gerrit-Reviewer: jenkins-bot <>