jenkins-bot has submitted this change and it was merged.
Change subject: test pwb: load default encoding from config
......................................................................
test pwb: load default encoding from config
Partial revert of 1e54a7d
Set envvar PYTHONIOENCODING to console_encoding
Add pwb tests for environment and unicode
Use str() on pwb tests env vars
Use with syntax to prevent unclosed file in Python 3
Change-Id: Id8a6c7b9b9ea29c0b92c8eb14f047bd792e20e24
---
M pwb.py
M pywikibot/config2.py
M pywikibot/tools/__init__.py
M tests/__init__.py
M tests/aspects.py
A tests/pwb/print_env.py
M tests/pwb/print_locals.py
A tests/pwb/print_unicode.py
M tests/pwb_tests.py
M tests/utils.py
10 files changed, 151 insertions(+), 60 deletions(-)
Approvals:
John Vandenberg: Looks good to me, but someone else must approve
XZise: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pwb.py b/pwb.py
index 365a22e..b1bb52c 100644
--- a/pwb.py
+++ b/pwb.py
@@ -79,7 +79,8 @@
sys.path[0] = os.path.dirname(filename)
try:
- source = open(filename).read()
+ with open(filename, 'rb') as f:
+ source = f.read()
exec(compile(source, filename, "exec", dont_inherit=True),
main_mod.__dict__)
finally:
@@ -92,6 +93,16 @@
pwb.argvu = old_argvu
# end of snippet from coverage
+
+
+def abspath(path):
+ """Convert path to absolute path, with uppercase drive letter on
win32."""
+ path = os.path.abspath(path)
+ if path[0] != '/':
+ # normalise Windows drive letter
+ path = path[0].upper() + path[1:]
+ return path
+
if sys.version_info[0] not in (2, 3):
raise RuntimeError("ERROR: Pywikibot only runs under Python 2 "
@@ -107,9 +118,7 @@
# Establish a normalised path for the directory containing pwb.py.
# Either it is '.' if the user's current working directory is the same,
# or it is the absolute path for the directory of pwb.py
-absolute_path = os.path.dirname(sys.argv[0])
-if not os.path.isabs(absolute_path):
- absolute_path = os.path.abspath(os.path.join(os.curdir, absolute_path))
+absolute_path = abspath(os.path.dirname(sys.argv[0]))
rewrite_path = absolute_path
sys.path = [sys.path[0], rewrite_path,
@@ -157,7 +166,10 @@
# If successful, user-config.py already exists in one of the candidate
# directories. See config2.py for details on search order.
# Use env var to communicate to config2.py pwb.py location (bug 72918).
- os.environ['PYWIKIBOT2_DIR_PWB'] = os.path.split(__file__)[0]
+ _pwb_dir = os.path.split(__file__)[0]
+ if sys.platform == 'win32' and sys.version_info[0] < 3:
+ _pwb_dir = str(_pwb_dir)
+ os.environ[str('PYWIKIBOT2_DIR_PWB')] = _pwb_dir
import pywikibot # noqa
except RuntimeError as err:
# user-config.py to be created
@@ -195,10 +207,10 @@
# This is a rough (and quick!) emulation of 'package name' detection.
# a much more detailed implementation is in coverage's find_module.
#
https://bitbucket.org/ned/coveragepy/src/default/coverage/execfile.py
- cwd = os.path.abspath(os.getcwd())
+ cwd = abspath(os.getcwd())
if absolute_path == cwd:
- absolute_filename = os.path.abspath(filename)
- if absolute_filename.startswith(rewrite_path):
+ absolute_filename = abspath(filename)[:len(cwd)]
+ if absolute_filename == cwd:
relative_filename = os.path.relpath(filename)
# remove the filename, and use '.' instead of path separator.
file_package = os.path.dirname(
diff --git a/pywikibot/config2.py b/pywikibot/config2.py
index db23c5e..730d4dd 100644
--- a/pywikibot/config2.py
+++ b/pywikibot/config2.py
@@ -46,8 +46,6 @@
from warnings import warn
-from pywikibot.tools import default_encoding
-
# This frozen set should contain all imported modules/variables, so it must
# occur directly after the imports. At that point globals() only contains the
# names and some magic variables (like __name__)
@@ -346,7 +344,10 @@
# This default code should work fine, so you don't have to think about it.
# TODO: consider getting rid of this config variable.
try:
- console_encoding = sys.stdout.encoding
+ if sys.version_info[0] > 2 or not sys.stdout.encoding:
+ console_encoding = sys.stdout.encoding
+ else:
+ console_encoding = sys.stdout.encoding.decode('ascii')
except:
# When using pywikibot inside a daemonized twisted application,
# we get "StdioOnnaStick instance has no attribute 'encoding'"
@@ -933,7 +934,11 @@
_ConfigurationDeprecationWarning)
# Fix up default console_encoding
-console_encoding = default_encoding(console_encoding)
+if console_encoding is None:
+ if sys.platform == 'win32':
+ console_encoding = 'cp850'
+ else:
+ console_encoding = 'iso-8859-1'
# Fix up transliteration_target
if transliteration_target == 'not set':
diff --git a/pywikibot/tools/__init__.py b/pywikibot/tools/__init__.py
index 67e0087..22d6c4d 100644
--- a/pywikibot/tools/__init__.py
+++ b/pywikibot/tools/__init__.py
@@ -405,26 +405,6 @@
self.stop()
-def stream_encoding(stream):
- """Get encoding of the stream and use a default if not
existent/None."""
- try:
- encoding = stream.encoding
- except AttributeError:
- encoding = None
- return default_encoding(encoding)
-
-
-def default_encoding(encoding):
- """Return an encoding even if it's originally
None."""
- if encoding is None:
- if sys.platform == 'win32':
- return 'cp850'
- else:
- return 'iso-8859-1'
- else:
- return encoding
-
-
def itergroup(iterable, size):
"""Make an iterator that returns lists of (up to) size items from
iterable.
diff --git a/tests/__init__.py b/tests/__init__.py
index 031b90d..3f213d2 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -191,7 +191,8 @@
# overridden here to restrict retries to only 1, so developer builds fail more
# frequently in code paths resulting from mishandled server problems.
if config.max_retries > 2:
- print('max_retries reduced from %d to 1 for tests' % config.max_retries)
+ if 'PYWIKIBOT_TEST_QUIET' not in os.environ:
+ print('tests: max_retries reduced from %d to 1' % config.max_retries)
config.max_retries = 1
cache_misses = 0
diff --git a/tests/aspects.py b/tests/aspects.py
index 78cd606..2458b6b 100644
--- a/tests/aspects.py
+++ b/tests/aspects.py
@@ -1157,14 +1157,17 @@
self.orig_pywikibot_dir = None
if 'PYWIKIBOT2_DIR' in os.environ:
self.orig_pywikibot_dir = os.environ['PYWIKIBOT2_DIR']
- os.environ['PYWIKIBOT2_DIR'] = pywikibot.config.base_dir
+ base_dir = pywikibot.config.base_dir
+ if sys.platform == 'win32' and sys.version_info[0] < 3:
+ base_dir = str(base_dir)
+ os.environ[str('PYWIKIBOT2_DIR')] = base_dir
def tearDown(self):
"""Restore the environment after running the pwb.py
script."""
super(PwbTestCase, self).tearDown()
del os.environ['PYWIKIBOT2_DIR']
if self.orig_pywikibot_dir:
- os.environ['PYWIKIBOT2_DIR'] = self.orig_pywikibot_dir
+ os.environ[str('PYWIKIBOT2_DIR')] = self.orig_pywikibot_dir
def _execute(self, args, data_in=None, timeout=0, error=None):
from tests.utils import execute_pwb
diff --git a/tests/pwb/print_env.py b/tests/pwb/print_env.py
new file mode 100644
index 0000000..e61230d
--- /dev/null
+++ b/tests/pwb/print_env.py
@@ -0,0 +1,31 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""Script that forms part of pwb_tests."""
+from __future__ import unicode_literals
+
+import os
+import sys
+
+_pwb_dir = os.path.abspath(os.path.join(
+ os.path.split(__file__)[0], '..', '..'))
+_pwb_dir = _pwb_dir[0].upper() + _pwb_dir[1:]
+
+print('os.environ:')
+for k, v in sorted(os.environ.items()):
+ # Don't leak the password into logs
+ if k == 'USER_PASSWORD':
+ continue
+ # This only appears in subprocesses
+ if k in ['PYWIKIBOT2_DIR_PWB']:
+ continue
+ print("%r: %r" % (k, v))
+
+print('sys.path:')
+for path in sys.path:
+ if path == '' or path.startswith('.'):
+ continue
+ # Normalise DOS drive letter
+ path = path[0].upper() + path[1:]
+ if path.startswith(_pwb_dir):
+ continue
+ print(path)
diff --git a/tests/pwb/print_locals.py b/tests/pwb/print_locals.py
index c39b776..261327a 100644
--- a/tests/pwb/print_locals.py
+++ b/tests/pwb/print_locals.py
@@ -1,3 +1,5 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
"""Script that forms part of pwb_tests."""
from __future__ import unicode_literals
diff --git a/tests/pwb/print_unicode.py b/tests/pwb/print_unicode.py
new file mode 100644
index 0000000..a8fa757
--- /dev/null
+++ b/tests/pwb/print_unicode.py
@@ -0,0 +1,9 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""Script that forms part of pwb_tests."""
+from __future__ import unicode_literals
+
+import pywikibot
+
+pywikibot.output('Häuser')
+print('Häuser')
diff --git a/tests/pwb_tests.py b/tests/pwb_tests.py
index fe52837..80bcf87 100644
--- a/tests/pwb_tests.py
+++ b/tests/pwb_tests.py
@@ -22,9 +22,7 @@
from tests.utils import execute, execute_pwb
from tests.aspects import unittest, PwbTestCase
-testbasepath = os.path.join(_tests_dir, 'pwb')
-print_locals_test_package = 'tests.pwb.print_locals'
-print_locals_test_script = os.path.join(testbasepath, 'print_locals.py')
+_pwb_tests_dir = os.path.join(_tests_dir, 'pwb')
class TestPwb(PwbTestCase):
@@ -42,17 +40,44 @@
site = False
net = False
- def testScriptEnvironment(self):
+ def _do_check(self, name):
+ package_name = 'tests.pwb.' + name
+ script_path = os.path.join(_pwb_tests_dir, name + '.py')
+
+ direct = execute([sys.executable, '-m', package_name])
+ vpwb = execute_pwb([script_path])
+ self.maxDiff = None
+ self.assertEqual(direct['stdout'], vpwb['stdout'])
+
+ return (direct, vpwb)
+
+ def test_env(self):
"""
- Test environment of pywikibot.
+ Test external environment of pywikibot.
Make sure the environment is not contaminated, and is the same as
the environment we get when directly running a script.
"""
- direct = execute([sys.executable, '-m',
'tests.pwb.print_locals'])
- vpwb = execute_pwb([print_locals_test_script])
- self.maxDiff = None
- self.assertEqual(direct['stdout'], vpwb['stdout'])
+ self._do_check('print_env')
+
+ def test_locals(self):
+ """
+ Test internal environment of pywikibot.
+
+ Make sure the environment is not contaminated, and is the same as
+ the environment we get when directly running a script.
+ """
+ self._do_check('print_locals')
+
+ def test_unicode(self):
+ """Test printing unicode in pywikibot."""
+ (direct, vpwb) = self._do_check('print_unicode')
+
+ self.assertEqual('Häuser', direct['stdout'].strip())
+ self.assertEqual('Häuser', direct['stderr'].strip())
+ self.assertEqual('Häuser', vpwb['stdout'].strip())
+ self.assertEqual('Häuser', vpwb['stderr'].strip())
+
if __name__ == "__main__":
unittest.main(verbosity=10)
diff --git a/tests/utils.py b/tests/utils.py
index d23fd91..9833916 100644
--- a/tests/utils.py
+++ b/tests/utils.py
@@ -16,7 +16,9 @@
from warnings import warn
import pywikibot
-from pywikibot.tools import SelfCallDict, stream_encoding
+
+from pywikibot import config
+from pywikibot.tools import SelfCallDict
from pywikibot.site import Namespace
from pywikibot.data.api import CachedRequest
from pywikibot.data.api import Request as _original_Request
@@ -259,16 +261,24 @@
# Any environment variables added on Windows must be of type
# str() on Python 2.
env = os.environ.copy()
+
+ # Prevent output by test package; e.g. 'max_retries reduced from x to y'
+ env[str('PYWIKIBOT_TEST_QUIET')] = str('1')
+
# sys.path may have been modified by the test runner to load dependencies.
- env['PYTHONPATH'] = ":".join(sys.path)
+ pythonpath = os.pathsep.join(sys.path)
+ if sys.platform == 'win32' and sys.version_info[0] < 3:
+ pythonpath = str(pythonpath)
+ env[str('PYTHONPATH')] = pythonpath
+ env[str('PYTHONIOENCODING')] = str(config.console_encoding)
+
# LC_ALL is used by i18n.input as an alternative for userinterface_lang
if pywikibot.config.userinterface_lang:
- env['LC_ALL'] = str(pywikibot.config.userinterface_lang)
+ env[str('LC_ALL')] = str(pywikibot.config.userinterface_lang)
+
# Set EDITOR to an executable that ignores all arguments and does nothing.
- if sys.platform == 'win32':
- env['EDITOR'] = str('call')
- else:
- env['EDITOR'] = 'true'
+ env[str('EDITOR')] = str('call' if sys.platform == 'win32'
else 'true')
+
options = {
'stdout': subprocess.PIPE,
'stderr': subprocess.PIPE
@@ -276,14 +286,27 @@
if data_in is not None:
options['stdin'] = subprocess.PIPE
- p = subprocess.Popen(command, env=env, **options)
-
- stdin_encoding = stream_encoding(p.stdin)
- stdout_encoding = stream_encoding(p.stdout)
- stderr_encoding = stream_encoding(p.stderr)
+ try:
+ p = subprocess.Popen(command, env=env, **options)
+ except TypeError:
+ # Generate a more informative error
+ if sys.platform == 'win32' and sys.version_info[0] < 3:
+ unicode_env = [(k, v) for k, v in os.environ.items()
+ if not isinstance(k, str) or
+ not isinstance(v, str)]
+ if unicode_env:
+ raise TypeError('os.environ must contain only str: %r'
+ % unicode_env)
+ child_unicode_env = [(k, v) for k, v in env.items()
+ if not isinstance(k, str) or
+ not isinstance(v, str)]
+ if child_unicode_env:
+ raise TypeError('os.environ must contain only str: %r'
+ % child_unicode_env)
+ raise
if data_in is not None:
- p.stdin.write(data_in.encode(stdin_encoding))
+ p.stdin.write(data_in.encode(config.console_encoding))
p.stdin.flush() # _communicate() otherwise has a broken pipe
stderr_lines = b''
@@ -296,7 +319,7 @@
if error:
line = p.stderr.readline()
stderr_lines += line
- if error in line.decode(stdout_encoding):
+ if error in line.decode(config.console_encoding):
break
time.sleep(1)
waited += 1
@@ -309,8 +332,8 @@
data_out = p.communicate()
return {'exit_code': p.returncode,
- 'stdout': data_out[0].decode(stdout_encoding),
- 'stderr': (stderr_lines + data_out[1]).decode(stderr_encoding)}
+ 'stdout': data_out[0].decode(config.console_encoding),
+ 'stderr': (stderr_lines +
data_out[1]).decode(config.console_encoding)}
def execute_pwb(args, data_in=None, timeout=0, error=None):
--
To view, visit
https://gerrit.wikimedia.org/r/204018
To unsubscribe, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Id8a6c7b9b9ea29c0b92c8eb14f047bd792e20e24
Gerrit-PatchSet: 13
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>