jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/606415 )
Change subject: [IMPR] Add explicit return value for _query_continue and _continue
......................................................................
[IMPR] Add explicit return value for _query_continue and _continue
- Add explicit return value for thes QueryGenerator methods.
_query_continue and _continue are called in line 2889 and
expects a boolean result; the function is assigned in 2471
and 2474. But only _query_continue returns True only for
one part of processing. All other cases return implicit
None but an explicit False should be returned instead.
- return None in Request._simulate as last resort
- Futhermore remove PY2 related code
Change-Id: I36340e56ab18a7e78202bc8a314fca9f8958c353
---
M pywikibot/data/api.py
1 file changed, 70 insertions(+), 112 deletions(-)
Approvals:
Zhuyifei1999: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index 4b421dc..217fc51 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -5,28 +5,24 @@
#
# Distributed under the terms of the MIT license.
#
-from __future__ import absolute_import, division, unicode_literals
-
import datetime
import hashlib
import inspect
import json
import os
+import pickle
import pprint
import re
import traceback
-try:
- from collections.abc import Container, MutableMapping, Sized
-except ImportError: # Python 2.7
- from collections import Container, MutableMapping, Sized
-from email.mime.nonmultipart import MIMENonMultipart
-from warnings import warn
-try:
- import cPickle as pickle # noqa: N813
-except ImportError:
- import pickle
+from collections.abc import Container, MutableMapping, Sized
+from email.generator import BytesGenerator
+from email.mime.multipart import MIMEMultipart as MIMEMultipartOrig
+from email.mime.nonmultipart import MIMENonMultipart
+from io import BytesIO
+from warnings import warn
+from urllib.parse import urlencode, unquote
import pywikibot
@@ -39,62 +35,11 @@
)
from pywikibot.family import SubdomainFamily
from pywikibot.tools import (
- deprecated, itergroup, PY2, PYTHON_VERSION,
- getargspec, UnicodeType, remove_last_args
+ deprecated, itergroup, PYTHON_VERSION,
+ getargspec, remove_last_args
)
from pywikibot.tools.formatter import color_format
-if not PY2:
- from urllib.parse import urlencode, unquote
-
- # Bug: T113120, T228841
- # Subclassing necessary to fix bug of the email package in Python 3:
- # see https://bugs.python.org/issue19003
- # see https://bugs.python.org/issue18886
- # The following solution might be removed if the bug is fixed for
- # Python versions which are supported by PWB.
-
- from email.generator import BytesGenerator
- from email.mime.multipart import MIMEMultipart as MIMEMultipartOrig
- from io import BytesIO
-
- class CTEBinaryBytesGenerator(BytesGenerator):
-
- """Workaround for bug in python 3 email handling of CTE binary."""
-
- def __init__(self, *args, **kwargs):
- """Initializer."""
- super(CTEBinaryBytesGenerator, self).__init__(*args, **kwargs)
- self._writeBody = self._write_body
-
- def _write_body(self, msg):
- if msg['content-transfer-encoding'] == 'binary':
- self._fp.write(msg.get_payload(decode=True))
- else:
- super(CTEBinaryBytesGenerator, self)._handle_text(msg)
-
- class CTEBinaryMIMEMultipart(MIMEMultipartOrig):
-
- """Workaround for bug in python 3 email handling of CTE binary."""
-
- def as_bytes(self, unixfrom=False, policy=None):
- """Return unmodified binary payload."""
- policy = self.policy if policy is None else policy
- fp = BytesIO()
- g = CTEBinaryBytesGenerator(fp, mangle_from_=False, policy=policy)
- g.flatten(self, unixfrom=unixfrom)
- return fp.getvalue()
-
- MIMEMultipart = CTEBinaryMIMEMultipart
-else:
- from urllib import urlencode, unquote
- from email.mime.multipart import MIMEMultipart
-
- # Bug: T243710 (Python 2)
- # see https://github.com/jxtech/wechatpy/issues/375
- from urllib import quote
- quote(b'non-empty-string', safe=b'')
-
_logger = 'data.api'
@@ -117,6 +62,45 @@
http.cookie_jar.clear(cookie.domain, cookie.path, cookie.name)
+# Bug: T113120, T228841
+# Subclassing necessary to fix bug of the email package in Python 3:
+# see https://bugs.python.org/issue19003
+# see https://bugs.python.org/issue18886
+# The following solution might be removed if the bug is fixed for
+# Python versions which are supported by PWB, probably with Python 3.5
+
+class CTEBinaryBytesGenerator(BytesGenerator):
+
+ """Workaround for bug in python 3 email handling of CTE binary."""
+
+ def __init__(self, *args, **kwargs):
+ """Initializer."""
+ super(CTEBinaryBytesGenerator, self).__init__(*args, **kwargs)
+ self._writeBody = self._write_body
+
+ def _write_body(self, msg):
+ if msg['content-transfer-encoding'] == 'binary':
+ self._fp.write(msg.get_payload(decode=True))
+ else:
+ super(CTEBinaryBytesGenerator, self)._handle_text(msg)
+
+
+class CTEBinaryMIMEMultipart(MIMEMultipartOrig):
+
+ """Workaround for bug in python 3 email handling of CTE binary."""
+
+ def as_bytes(self, unixfrom=False, policy=None):
+ """Return unmodified binary payload."""
+ policy = self.policy if policy is None else policy
+ fp = BytesIO()
+ g = CTEBinaryBytesGenerator(fp, mangle_from_=False, policy=policy)
+ g.flatten(self, unixfrom=unixfrom)
+ return fp.getvalue()
+
+
+MIMEMultipart = CTEBinaryMIMEMultipart
+
+
class APIError(Error):
"""The wiki site returned an error message."""
@@ -126,7 +110,7 @@
self.code = code
self.info = info
self.other = kwargs
- self.unicode = UnicodeType(self.__str__())
+ self.unicode = self.__str__()
def __repr__(self):
"""Return internal representation."""
@@ -330,7 +314,7 @@
assert('mime' in result['help'])
assert(result['help']['mime'] == 'text/plain')
assert('help' in result['help'])
- assert(isinstance(result['help']['help'], UnicodeType))
+ assert(isinstance(result['help']['help'], str))
help_text = result['help']['help']
@@ -374,7 +358,7 @@
@type modules: iterable or basestring
@rtype: set
"""
- if isinstance(modules, UnicodeType):
+ if isinstance(modules, str):
return set(modules.split('|'))
return set(modules)
@@ -1065,9 +1049,9 @@
"""A request to a Site's api.php interface.
Attributes of this object (except for the special parameters listed
- below) get passed as commands to api.php, and can be get or set using
- the dict interface. All attributes must be strings (or unicode). Use
- an empty string for parameters that don't require a value. For example,
+ below) get passed as commands to api.php, and can be get or set
+ using the dict interface. All attributes must be strings. Use an
+ empty string for parameters that don't require a value. For example,
Request(action="query", titles="Foo bar", prop="info", redirects="")
corresponds to the API request
"api.php?action=query&titles=Foo%20bar&prop=info&redirects"
@@ -1353,8 +1337,7 @@
* datetime.datetime (using strftime and ISO8601 format)
* pywikibot.page.BasePage (using title (+namespace; -section))
- All other datatypes are converted to string using unicode() on Python 2
- and str() on Python 3.
+ All other datatypes are converted to string.
"""
if isinstance(value, datetime.datetime):
return value.strftime(pywikibot.Timestamp.ISO8601Format)
@@ -1362,7 +1345,7 @@
assert(value.site == self.site)
return value.title(with_section=False)
else:
- return UnicodeType(value)
+ return str(value)
def __getitem__(self, key):
"""Implement dict interface."""
@@ -1379,15 +1362,14 @@
@type value: str in site encoding
(string types may be a `|`-separated list)
- iterable, where items are converted to unicode
+ iterable, where items are converted to string
with special handling for datetime.datetime to convert it to a
string using the ISO 8601 format accepted by the MediaWiki API.
"""
- # Allow site encoded bytes (note: str is a subclass of bytes in py2)
if isinstance(value, bytes):
value = value.decode(self.site.encoding())
- if isinstance(value, UnicodeType):
+ if isinstance(value, str):
value = value.split('|')
if hasattr(value, 'api_iter'):
@@ -1542,10 +1524,6 @@
# which is not a superset of ascii may be problematic.
try:
value.encode('ascii')
- # In Python 2, ascii API params should be represented as 'foo'
- # rather than u'foo'
- if PY2:
- value = str(value)
except UnicodeError:
try:
value = value.encode(self.site.encoding())
@@ -1553,10 +1531,7 @@
pywikibot.error(
"_encoded_items: '%s' could not be encoded as '%s':"
' %r' % (key, self.site.encoding(), value))
- if PY2:
- key = key.encode('ascii')
- else:
- assert key.encode('ascii')
+ assert key.encode('ascii')
assert isinstance(key, str)
params[key] = value
return params
@@ -1596,6 +1571,7 @@
if config.simulate is not True:
pywikibot.sleep(float(config.simulate))
return {action: {'result': 'Success', 'nochange': ''}}
+ return None
def _is_wikibase_error_retryable(self, error):
ERR_MSG = (
@@ -1687,10 +1663,7 @@
container.attach(submsg)
# strip the headers to get the HTTP message body
- if not PY2:
- body = container.as_bytes()
- else:
- body = container.as_string()
+ body = container.as_bytes()
marker = b'\n\n' # separates headers from body
eoh = body.find(marker)
body = body[eoh + len(marker):]
@@ -1767,7 +1740,7 @@
@raises APIError: unknown action found
@raises APIError: unknown query result type
"""
- if not isinstance(data, UnicodeType):
+ if not isinstance(data, str):
data = data.decode(self.site.encoding())
pywikibot.debug(('API response received from {}:\n'
.format(self.site)) + data, _logger)
@@ -1897,12 +1870,7 @@
pywikibot.error('Detected MediaWiki API exception {}{}'
.format(e, '; retrying' if retry else '; raising'))
- # Due to bug T66958, Page's repr may return non ASCII bytes
- # Get as bytes in PY2 and decode with the console encoding as
- # the rest should be ASCII anyway.
param_repr = str(self._params)
- if PY2:
- param_repr = param_repr.decode(config.console_encoding)
pywikibot.log('MediaWiki exception {} details:\n'
' query=\n{}\n'
' response=\n{}'
@@ -2020,7 +1988,7 @@
if key in ('error', 'warnings'):
continue
assert key not in error
- assert isinstance(result[key], UnicodeType), \
+ assert isinstance(result[key], str), \
'Unexpected %s: %r' % (key, result[key])
error[key] = result[key]
@@ -2102,12 +2070,7 @@
# raise error
try:
- # Due to bug T66958, Page's repr may return non ASCII bytes
- # Get as bytes in PY2 and decode with the console encoding as
- # the rest should be ASCII anyway.
param_repr = str(self._params)
- if PY2:
- param_repr = param_repr.decode(config.console_encoding)
pywikibot.log('API Error: query=\n%s'
% pprint.pformat(param_repr))
pywikibot.log(' response=\n%s'
@@ -2698,7 +2661,7 @@
return False
- if isinstance(namespaces, UnicodeType):
+ if isinstance(namespaces, str):
namespaces = namespaces.split('|')
# Use Namespace id (int) here; Request will cast int to str
@@ -2727,11 +2690,14 @@
"Missing '%s' key(s) in ['%s'] value."
% (self.continuekey, self.continue_name))
return True
+
for query_continue_pair in self.data['query-continue'].values():
self._add_continues(query_continue_pair)
+ return False # a new request with query-continue is needed
def _continue(self):
self._add_continues(self.data['continue'])
+ return False # a new request with continue is needed
def _add_continues(self, continue_pair):
for key, value in continue_pair.items():
@@ -2872,7 +2838,7 @@
if 'query' not in self.data:
pywikibot.log("%s: 'query' not found in api response." %
self.__class__.__name__)
- pywikibot.log(UnicodeType(self.data))
+ pywikibot.log(str(self.data))
# if (query-)continue is present, self.resultkey might not have
# been fetched yet
if self.continue_name not in self.data:
@@ -3056,7 +3022,7 @@
if isinstance(v, list):
old_dict[k].extend(v)
continue
- assert isinstance(v, (UnicodeType, int)), (
+ assert isinstance(v, (str, int)), (
'continued API result had an unexpected type: %s' % type(v))
@@ -3286,14 +3252,6 @@
if hasattr(query, 'items'):
query = list(query.items())
- if PY2:
- def _encode(x):
- if isinstance(x, UnicodeType):
- return x.encode('utf-8')
- else:
- return x
- query = [(pair[0], _encode(pair[1])) for pair in query]
-
# parameters ending on 'token' should go last
# wpEditToken should go very last
query.sort(key=lambda x: x[0].lower().endswith('token')
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/606415
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I36340e56ab18a7e78202bc8a314fca9f8958c353
Gerrit-Change-Number: 606415
Gerrit-PatchSet: 6
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com>
Gerrit-Reviewer: Zhuyifei1999 <zhuyifei1999(a)gmail.com>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/610062 )
Change subject: [4.0] Remove code duplication of pwb.py and setup.py
......................................................................
[4.0] Remove code duplication of pwb.py and setup.py
- Remove duplicate code or Python version check:
pwb.py imports mandatory or script dependecies from setup.py
and setup.py check the Python version supported.
Re-use this check in pwb.py
- Remove PY2-related code
- Additional doc for check_modules
- import Path after version checking
Bug: T239542
Bug: T213287
Change-Id: I476a42efc439f4fed5d10888ece76d73b294d0c8
---
M pwb.py
M setup.py
2 files changed, 25 insertions(+), 44 deletions(-)
Approvals:
Zhuyifei1999: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pwb.py b/pwb.py
index e3711e0..3880526 100755
--- a/pwb.py
+++ b/pwb.py
@@ -18,8 +18,7 @@
#
# Distributed under the terms of the MIT license.
#
-from __future__ import (absolute_import, division,
- print_function, unicode_literals)
+from __future__ import absolute_import, print_function, unicode_literals
from difflib import get_close_matches
from importlib import import_module
@@ -30,36 +29,9 @@
from warnings import warn
-PYTHON_VERSION = sys.version_info[:3]
-PY2 = (PYTHON_VERSION[0] == 2)
-
-if not PY2:
- from pathlib import Path
-else:
- from pathlib2 import Path
-
-
-VERSIONS_REQUIRED_MESSAGE = """
-Pywikibot is not available on:
-{version}
-
-This version of Pywikibot only supports Python 2.7.4+ or 3.4+.
-"""
-
-
-def python_is_supported():
- """Check that Python is supported."""
- # Any change to this must be copied to setup.py
- return PYTHON_VERSION >= (3, 4, 0) or PY2 and PYTHON_VERSION >= (2, 7, 4)
-
-
-if not python_is_supported():
- print(VERSIONS_REQUIRED_MESSAGE.format(version=sys.version))
- sys.exit(1)
pwb = None
-
# The following snippet was developed by Ned Batchelder (and others)
# for coverage [1], with python 3 support [2] added later,
# and is available under the BSD license (see [3])
@@ -70,6 +42,7 @@
# [3]
# https://bitbucket.org/ned/coveragepy/src/2c5fb3a8b81c/setup.py?at=default#c…
+
def run_python_file(filename, argv, argvu, package=None):
"""Run a python file as if it were the main program on the command line.
@@ -79,17 +52,12 @@
"""
# Create a module to serve as __main__
old_main_mod = sys.modules['__main__']
- # it's explicitly using str() to bypass unicode_literals in Python 2
- main_mod = types.ModuleType(str('__main__'))
+ main_mod = types.ModuleType('__main__')
sys.modules['__main__'] = main_mod
main_mod.__file__ = filename
- if not PY2:
- main_mod.__builtins__ = sys.modules['builtins']
- else:
- main_mod.__builtins__ = sys.modules['__builtin__']
+ main_mod.__builtins__ = sys.modules['builtins']
if package:
- # it's explicitly using str() to bypass unicode_literals in Python 2
- main_mod.__package__ = str(package)
+ main_mod.__package__ = package
# Set sys.argv and the first path element properly.
old_argv = sys.argv
@@ -163,7 +131,16 @@
def check_modules(script=None):
- """Check whether mandatory modules are present."""
+ """Check whether mandatory modules are present.
+
+ This also checks Python version when importing deptendencies from setup.py
+
+ @param script: The script name to be checked for dependencies
+ @type script: str or None
+ @return: True if all dependencies are installed
+ @rtype: bool
+ @raise RuntimeError: wrong Python version found in setup.py
+ """
import pkg_resources
if script:
from setup import script_deps
@@ -201,9 +178,15 @@
return not missing_requirements
-if not check_modules():
+try:
+ if not check_modules():
+ raise RuntimeError('') # no further output needed
+except RuntimeError as e:
+ print(e)
sys.exit()
+from pathlib import Path # noqa: E402
+
filename, script_args, global_args = handle_args(*sys.argv)
# Search for user-config.py before creating one.
@@ -211,8 +194,6 @@
# directories. See config2.py for details on search order.
# Use env var to communicate to config2.py pwb.py location (bug T74918).
_pwb_dir = os.path.split(__file__)[0]
-if sys.platform == 'win32' and PY2:
- _pwb_dir = str(_pwb_dir)
os.environ['PYWIKIBOT_DIR_PWB'] = _pwb_dir
try:
import pywikibot as pwb
diff --git a/setup.py b/setup.py
index fe41208..9ecd187 100644
--- a/setup.py
+++ b/setup.py
@@ -36,7 +36,7 @@
PYTHON_VERSION = sys.version_info[:3]
-versions_required_message = """
+VERSIONS_REQUIRED_MESSAGE = """
Pywikibot is not available on:
{version}
@@ -46,12 +46,12 @@
def python_is_supported():
"""Check that Python is supported."""
- # Any change to this must be copied to pwb.py
return PYTHON_VERSION >= (3, 5, 0)
if not python_is_supported():
- raise RuntimeError(versions_required_message.format(version=sys.version))
+ # pwb.py checks this exception
+ raise RuntimeError(VERSIONS_REQUIRED_MESSAGE.format(version=sys.version))
# ------- setup extra_requires ------- #
extra_deps = {
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/610062
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I476a42efc439f4fed5d10888ece76d73b294d0c8
Gerrit-Change-Number: 610062
Gerrit-PatchSet: 4
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Zhuyifei1999 <zhuyifei1999(a)gmail.com>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/592487 )
Change subject: [tests] Print the exception tested with assertNoSite
......................................................................
[tests] Print the exception tested with assertNoSite
assertNoSite expects several exception when trying to create a
MWSite from url. But is is not clear, why creating the MWSite
fails. This can be happen if a urls doesn't exist any longer.
Therefore print the exception with this tests.
Change-Id: I80fcdfb68a15f75fe0650af83814c0e52e335e32
---
M tests/site_detect_tests.py
1 file changed, 6 insertions(+), 2 deletions(-)
Approvals:
Zhuyifei1999: Looks good to me, approved
jenkins-bot: Verified
diff --git a/tests/site_detect_tests.py b/tests/site_detect_tests.py
index a5f6230..34ffaf5 100644
--- a/tests/site_detect_tests.py
+++ b/tests/site_detect_tests.py
@@ -15,6 +15,7 @@
from pywikibot.site_detect import MWSite
from pywikibot.tools import PY2
+from tests import unittest_print
from tests.aspects import unittest, TestCase, PatchingTestCase
from tests.utils import DrySite
@@ -51,8 +52,11 @@
@type url: str
@raises AssertionError: Site under url is MediaWiki powered
"""
- self.assertRaises((AttributeError, ConnectionError, RuntimeError,
- ServerError, Timeout), MWSite, url)
+ with self.assertRaises((AttributeError, ConnectionError, RuntimeError,
+ ServerError, Timeout)) as e:
+ MWSite(url)
+ unittest_print('\nassertNoSite expected exception:\n{e!r}'
+ .format(e=e.exception))
class StandardVersionSiteTestCase(SiteDetectionTestCase):
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/592487
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I80fcdfb68a15f75fe0650af83814c0e52e335e32
Gerrit-Change-Number: 592487
Gerrit-PatchSet: 2
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Dvorapa <dvorapa(a)seznam.cz>
Gerrit-Reviewer: Zhuyifei1999 <zhuyifei1999(a)gmail.com>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/592483 )
Change subject: [tests] Add flake8-color to provide a color format string
......................................................................
[tests] Add flake8-color to provide a color format string
This enables to easy find the flake8 error messages
Bug: T117570
Change-Id: Ibbe0a165d1f4caa13379728bc6513365bde9a946
---
M setup.py
M tox.ini
2 files changed, 3 insertions(+), 0 deletions(-)
Approvals:
Zhuyifei1999: Looks good to me, approved
jenkins-bot: Verified
diff --git a/setup.py b/setup.py
index 21711c2..efc9e49 100644
--- a/setup.py
+++ b/setup.py
@@ -59,6 +59,7 @@
'pydocstyle>=4.0.0;python_version>="3.4"',
'hacking',
'flake8-coding',
+ 'flake8-colors',
'flake8-comprehensions>=3.1.4;python_version>="3.8"',
'flake8-comprehensions>=2.2.0;python_version>="3.5"',
'flake8-comprehensions>=2.0.0,<2.2.0;python_version=="3.4"',
diff --git a/tox.ini b/tox.ini
index d34bd55..c1ad738 100644
--- a/tox.ini
+++ b/tox.ini
@@ -107,6 +107,8 @@
accept-encodings = utf-8
require-code = true
classmethod-decorators = classmethod,classproperty
+# flake8-color format string
+format = ${blue}%(path)s${reset}: ${bold}line %(row)d:${reset}%(col)d: ${red_bold}%(code)s${reset} %(text)s
# The following are to be fixed
# N802: function name should be lowercase
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/592483
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Ibbe0a165d1f4caa13379728bc6513365bde9a946
Gerrit-Change-Number: 592483
Gerrit-PatchSet: 2
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Dvorapa <dvorapa(a)seznam.cz>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: Zhuyifei1999 <zhuyifei1999(a)gmail.com>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged