jenkins-bot merged this change.
threadedhttp.py: Deprecate HttpRequest.content in favour of text
Bug: T173352
Change-Id: I2769f207fd297969c2de38996a6c741fd9f597a0
---
M pywikibot/comms/http.py
M pywikibot/comms/threadedhttp.py
M pywikibot/data/sparql.py
M pywikibot/pagegenerators.py
M pywikibot/proofreadpage.py
M pywikibot/site_detect.py
M pywikibot/version.py
M pywikibot/weblib.py
M scripts/flickrripper.py
M scripts/imagerecat.py
M scripts/reflinks.py
M tests/http_tests.py
M tests/sparql_tests.py
M tests/weblib_tests.py
14 files changed, 55 insertions(+), 46 deletions(-)
diff --git a/pywikibot/comms/http.py b/pywikibot/comms/http.py
index 76878ae..d98ab8a 100644
--- a/pywikibot/comms/http.py
+++ b/pywikibot/comms/http.py
@@ -14,7 +14,7 @@
from __future__ import absolute_import, print_function, unicode_literals
#
-# (C) Pywikibot team, 2007-2017
+# (C) Pywikibot team, 2007-2018
#
# Distributed under the terms of the MIT license.
#
@@ -303,7 +303,7 @@
issue_deprecation_warning(
'Invoking http.request without argument site', 'http.fetch()', 3)
r = fetch(uri, method, params, body, headers, **kwargs)
- return r.content
+ return r.text
baseuri = site.base_url(uri)
@@ -319,7 +319,7 @@
headers['user-agent'] = user_agent(site, format_string)
r = fetch(baseuri, method, params, body, headers, **kwargs)
- return r.content
+ return r.text
def get_authentication(uri):
diff --git a/pywikibot/comms/threadedhttp.py b/pywikibot/comms/threadedhttp.py
index 835fa50..4eadb57 100644
--- a/pywikibot/comms/threadedhttp.py
+++ b/pywikibot/comms/threadedhttp.py
@@ -3,7 +3,7 @@
from __future__ import absolute_import, unicode_literals
#
-# (C) Pywikibot team, 2007-2017
+# (C) Pywikibot team, 2007-2018
#
# Distributed under the terms of the MIT license.
#
@@ -22,7 +22,7 @@
import pywikibot
-from pywikibot.tools import UnicodeMixin
+from pywikibot.tools import deprecated, UnicodeMixin
_logger = "comm.threadedhttp"
@@ -189,13 +189,19 @@
return self.raw.decode(encoding, errors)
@property
+ @deprecated('the `text` property')
def content(self):
+ """DEPRECATED. Return the response decoded by the detected encoding."""
+ return self.text
+
+ @property
+ def text(self):
"""Return the response decoded by the detected encoding."""
return self.decode(self.encoding)
def __unicode__(self):
"""Return the response decoded by the detected encoding."""
- return self.content
+ return self.text
def __bytes__(self):
"""Return the undecoded response."""
diff --git a/pywikibot/data/sparql.py b/pywikibot/data/sparql.py
index 392f8a8..13ec4ca 100644
--- a/pywikibot/data/sparql.py
+++ b/pywikibot/data/sparql.py
@@ -144,10 +144,10 @@
while True:
try:
self.last_response = http.fetch(url, headers=headers)
- if not self.last_response.content:
+ if not self.last_response.text:
return None
try:
- return json.loads(self.last_response.content)
+ return json.loads(self.last_response.text)
except ValueError:
return None
except Timeout:
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index 55a5c1b..0392e0f 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -2961,7 +2961,7 @@
if 500 <= req.status < 600:
raise ServerError(
'received {0} status from {1}'.format(req.status, req.uri))
- j = json.loads(req.content)
+ j = json.loads(req.text)
raw_pages = j['*'][0]['a']['*']
for raw_page in raw_pages:
yield raw_page
diff --git a/pywikibot/proofreadpage.py b/pywikibot/proofreadpage.py
index 5b36e5b..44059f4 100644
--- a/pywikibot/proofreadpage.py
+++ b/pywikibot/proofreadpage.py
@@ -18,7 +18,7 @@
"""
#
-# (C) Pywikibot team, 2015-2017
+# (C) Pywikibot team, 2015-2018
#
# Distributed under the terms of the MIT license.
#
@@ -527,7 +527,7 @@
pywikibot.error('Error fetching HTML for %s.' % self)
raise
- soup = Soup(response.content)
+ soup = Soup(response.text)
try:
self._url_image = soup.find(class_='prp-page-image')
@@ -566,7 +566,7 @@
pywikibot.error('Querying %s: %s' % (cmd_uri, e))
return (True, e)
- data = json.loads(response.content)
+ data = json.loads(response.text)
assert 'error' in data, 'Error from phe-tools: %s' % data
assert data['error'] in [0, 1], 'Error from phe-tools: %s' % data
diff --git a/pywikibot/site_detect.py b/pywikibot/site_detect.py
index 1d82dea..a9dea68 100644
--- a/pywikibot/site_detect.py
+++ b/pywikibot/site_detect.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""Classes for detecting a MediaWiki site."""
#
-# (C) Pywikibot team, 2010-2015
+# (C) Pywikibot team, 2010-2018
#
# Distributed under the terms of the MIT license.
#
@@ -63,7 +63,7 @@
self.fromurl = fromurl
- data = r.content
+ data = r.text
wp = WikiHTMLPageParser(fromurl)
wp.feed(data)
@@ -118,7 +118,7 @@
response = fetch(
self.api +
"?action=query&meta=siteinfo&siprop=interwikimap&sifilteriw=local&format=json")
- iw = json.loads(response.content)
+ iw = json.loads(response.text)
if 'error' in iw:
raise RuntimeError('%s - %s' % (iw['error']['code'],
iw['error']['info']))
@@ -146,7 +146,7 @@
"""Extract the version from API help with ?version enabled."""
if self.version is None:
try:
- d = fetch(self.api + '?version&format=json').content
+ d = fetch(self.api + '?version&format=json').text
try:
d = json.loads(d)
except ValueError:
@@ -167,7 +167,7 @@
response = fetch(self.api + '?action=query&meta=siteinfo&format=json')
check_response(response)
# remove preleading newlines and Byte Order Mark (BOM), see T128992
- content = response.content.strip().lstrip('\uFEFF')
+ content = response.text.strip().lstrip('\uFEFF')
info = json.loads(content)
self.private_wiki = ('error' in info and
info['error']['code'] == 'readapidenied')
@@ -306,5 +306,5 @@
raise ServerError('Bad Gateway')
elif response.status == 500:
raise ServerError('Internal Server Error')
- elif response.status == 200 and SERVER_DB_ERROR_MSG in response.content:
+ elif response.status == 200 and SERVER_DB_ERROR_MSG in response.text:
raise ServerError('Server cannot access the database')
diff --git a/pywikibot/version.py b/pywikibot/version.py
index e671c90..77f4c34 100644
--- a/pywikibot/version.py
+++ b/pywikibot/version.py
@@ -383,7 +383,7 @@
url = 'https://gerrit.wikimedia.org/r/projects/pywikibot%2Fcore/branches/master'
# Gerrit API responses include )]}' at the beginning, make sure to strip it out
buf = http.fetch(uri=url,
- headers={'user-agent': '{pwb}'}).content[4:]
+ headers={'user-agent': '{pwb}'}).text[4:]
try:
hsh = json.loads(buf)['revision']
diff --git a/pywikibot/weblib.py b/pywikibot/weblib.py
index 3cf13c6..87494fa 100644
--- a/pywikibot/weblib.py
+++ b/pywikibot/weblib.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""Functions for manipulating external links or querying third-party sites."""
#
-# (C) Pywikibot team, 2013
+# (C) Pywikibot team, 2013-2018
#
# Distributed under the terms of the MIT license.
#
@@ -48,7 +48,7 @@
retry_count = 0
while retry_count <= config2.max_retries:
try:
- jsontext = http.fetch(uri).content
+ jsontext = http.fetch(uri).text
break
except RequestsConnectionError as e:
error = e
@@ -85,7 +85,7 @@
query['date'] = timestamp
uri = uri + urlencode(query)
- xmltext = http.fetch(uri).content
+ xmltext = http.fetch(uri).text
if "success" in xmltext:
data = ET.fromstring(xmltext)
return data.find('.//webcite_url').text
diff --git a/scripts/flickrripper.py b/scripts/flickrripper.py
index b03604d..3ef2fb2 100755
--- a/scripts/flickrripper.py
+++ b/scripts/flickrripper.py
@@ -168,7 +168,7 @@
parameters = urlencode({'id': photo_id, 'raw': 'on'})
return fetch(
- 'http://wikipedia.ramselehof.de/flinfo.php?%s' % parameters).content
+ 'http://wikipedia.ramselehof.de/flinfo.php?%s' % parameters).text
def getFilename(photoInfo, site=None, project=u'Flickr'):
diff --git a/scripts/imagerecat.py b/scripts/imagerecat.py
index ea4c5d2..884da46 100755
--- a/scripts/imagerecat.py
+++ b/scripts/imagerecat.py
@@ -166,7 +166,7 @@
commonsHelperPage = fetch(
"https://toolserver.org/~daniel/WikiSense/CommonSense.php?%s" % parameters)
matches = commonsenseRe.search(
- commonsHelperPage.content)
+ commonsHelperPage.text)
gotInfo = True
else:
break
@@ -229,7 +229,7 @@
while not gotInfo:
try:
page = fetch('https://nominatim.openstreetmap.org/reverse?format=xml&%s' % parameters)
- et = xml.etree.ElementTree.fromstring(page.content)
+ et = xml.etree.ElementTree.fromstring(page.text)
gotInfo = True
except IOError:
pywikibot.output(u'Got an IOError, let\'s try again')
diff --git a/scripts/reflinks.py b/scripts/reflinks.py
index b4356bc..d713f9f 100755
--- a/scripts/reflinks.py
+++ b/scripts/reflinks.py
@@ -462,7 +462,7 @@
pywikibot.output(u'PDF file.')
fd, infile = tempfile.mkstemp()
urlobj = os.fdopen(fd, 'w+')
- urlobj.write(f.content)
+ urlobj.write(f.text)
try:
pdfinfo_out = subprocess.Popen([r"pdfinfo", "/dev/stdin"],
diff --git a/tests/http_tests.py b/tests/http_tests.py
index f0829a7..e348f5a 100644
--- a/tests/http_tests.py
+++ b/tests/http_tests.py
@@ -48,8 +48,8 @@
r = http._enqueue('http://www.wikipedia.org/')
self.assertIsInstance(r, threadedhttp.HttpRequest)
self.assertEqual(r.status, 200)
- self.assertIn('<html lang="mul"', r.content)
- self.assertIsInstance(r.content, unicode)
+ self.assertIn('<html lang="mul"', r.text)
+ self.assertIsInstance(r.text, unicode)
self.assertIsInstance(r.raw, bytes)
def test_fetch(self):
@@ -57,8 +57,10 @@
r = http.fetch('http://www.wikipedia.org/')
self.assertIsInstance(r, threadedhttp.HttpRequest)
self.assertEqual(r.status, 200)
- self.assertIn('<html lang="mul"', r.content)
- self.assertIsInstance(r.content, unicode)
+ self.assertIn('<html lang="mul"', r.text)
+ self.assertIsInstance(r.text, unicode)
+ with suppress_warnings(r'.*HttpRequest\.content is deprecated'):
+ self.assertEqual(r.content, r.text)
self.assertIsInstance(r.raw, bytes)
@@ -150,7 +152,7 @@
response = http.fetch(
uri='https://testssl-expire-r2i2.disig.sk/index.en.html',
disable_ssl_certificate_validation=True)
- r = response.content
+ r = response.text
self.assertIsInstance(r, unicode)
self.assertTrue(re.search(r'<title>.*</title>', r))
http.session.close() # clear the connection
@@ -454,7 +456,7 @@
self.assertIsNone(req.charset)
self.assertEqual('latin1', req.encoding)
self.assertEqual(req.raw, CharsetTestCase.LATIN1_BYTES)
- self.assertEqual(req.content, CharsetTestCase.STR)
+ self.assertEqual(req.text, CharsetTestCase.STR)
def test_no_charset(self):
"""Test decoding without explicit charset."""
@@ -466,7 +468,7 @@
self.assertIsNone(req.charset)
self.assertEqual('latin1', req.encoding)
self.assertEqual(req.raw, CharsetTestCase.LATIN1_BYTES)
- self.assertEqual(req.content, CharsetTestCase.STR)
+ self.assertEqual(req.text, CharsetTestCase.STR)
def test_content_type_application_json_without_charset(self):
"""Test decoding without explicit charset but JSON content."""
@@ -537,7 +539,7 @@
self.assertIsNone(req.charset)
self.assertEqual('utf-8', req.encoding)
self.assertEqual(req.raw, CharsetTestCase.UTF8_BYTES)
- self.assertEqual(req.content, CharsetTestCase.STR)
+ self.assertEqual(req.text, CharsetTestCase.STR)
def test_same_charset(self):
"""Test decoding with explicit and equal charsets."""
@@ -545,7 +547,7 @@
self.assertEqual('utf-8', req.charset)
self.assertEqual('utf-8', req.encoding)
self.assertEqual(req.raw, CharsetTestCase.UTF8_BYTES)
- self.assertEqual(req.content, CharsetTestCase.STR)
+ self.assertEqual(req.text, CharsetTestCase.STR)
def test_header_charset(self):
"""Test decoding with different charsets and valid header charset."""
@@ -555,7 +557,7 @@
with patch('pywikibot.warning'):
self.assertEqual('utf-8', req.encoding)
self.assertEqual(req.raw, CharsetTestCase.UTF8_BYTES)
- self.assertEqual(req.content, CharsetTestCase.STR)
+ self.assertEqual(req.text, CharsetTestCase.STR)
def test_code_charset(self):
"""Test decoding with different charsets and invalid header charset."""
@@ -566,7 +568,7 @@
with patch('pywikibot.warning'):
self.assertEqual('latin1', req.encoding)
self.assertEqual(req.raw, CharsetTestCase.LATIN1_BYTES)
- self.assertEqual(req.content, CharsetTestCase.STR)
+ self.assertEqual(req.text, CharsetTestCase.STR)
def test_invalid_charset(self):
"""Test decoding with different and invalid charsets."""
@@ -579,7 +581,8 @@
UnicodeDecodeError, self.CODEC_CANT_DECODE_RE,
lambda: req.encoding)
self.assertEqual(req.raw, CharsetTestCase.LATIN1_BYTES)
- self.assertRaisesRegex(UnicodeDecodeError, self.CODEC_CANT_DECODE_RE, lambda: req.content)
+ self.assertRaisesRegex(
+ UnicodeDecodeError, self.CODEC_CANT_DECODE_RE, lambda: req.text)
class BinaryTestCase(TestCase):
@@ -645,7 +648,7 @@
r = http.fetch(uri=self.get_httpbin_url('/get'), params={})
self.assertEqual(r.status, 200)
- content = json.loads(r.content)
+ content = json.loads(r.text)
self.assertDictEqual(content['args'], {})
def test_unencoded_params(self):
@@ -658,7 +661,7 @@
r = http.fetch(uri=self.get_httpbin_url('/get'), params={'fish&chips': 'delicious'})
self.assertEqual(r.status, 200)
- content = json.loads(r.content)
+ content = json.loads(r.text)
self.assertDictEqual(content['args'], {'fish&chips': 'delicious'})
def test_encoded_params(self):
@@ -672,7 +675,7 @@
params={'fish%26chips': 'delicious'})
self.assertEqual(r.status, 200)
- content = json.loads(r.content)
+ content = json.loads(r.text)
self.assertDictEqual(content['args'], {'fish%26chips': 'delicious'})
@@ -686,8 +689,8 @@
r_body = http.fetch(uri=self.get_httpbin_url('/post'), method='POST',
body={'fish&chips': 'delicious'})
- self.assertDictEqual(json.loads(r_data.content),
- json.loads(r_body.content))
+ self.assertDictEqual(json.loads(r_data.text),
+ json.loads(r_body.text))
if __name__ == '__main__': # pragma: no cover
diff --git a/tests/sparql_tests.py b/tests/sparql_tests.py
index 933de95..838e567 100644
--- a/tests/sparql_tests.py
+++ b/tests/sparql_tests.py
@@ -87,7 +87,7 @@
def __init__(self, value):
"""Create container."""
- self.content = value
+ self.text = value
class TestSparql(WikidataTestCase):
diff --git a/tests/weblib_tests.py b/tests/weblib_tests.py
index e4b5414..fe4982a 100644
--- a/tests/weblib_tests.py
+++ b/tests/weblib_tests.py
@@ -1,7 +1,7 @@
# -*- coding: utf-8 -*-
"""Weblib test module."""
#
-# (C) Pywikibot team, 2014-2015
+# (C) Pywikibot team, 2014-2018
#
# Distributed under the terms of the MIT license.
#
@@ -35,7 +35,7 @@
def _test_response(self, response, *args, **kwargs):
# for later tests this is must be present, and it'll tell us the
# original content if that does not match
- self.assertIn('closest', response.content)
+ self.assertIn('closest', response.text)
def _get_archive_url(self, url, date_string=None):
with PatchedHttp(weblib, False) as p:
To view, visit change 421282. To unsubscribe, visit settings.