jenkins-bot has submitted this change and it was merged.

Change subject: test pwb: load default encoding from config ......................................................................

test pwb: load default encoding from config

Partial revert of 1e54a7d Set envvar PYTHONIOENCODING to console_encoding Add pwb tests for environment and unicode Use str() on pwb tests env vars Use with syntax to prevent unclosed file in Python 3

Change-Id: Id8a6c7b9b9ea29c0b92c8eb14f047bd792e20e24 --- M pwb.py M pywikibot/config2.py M pywikibot/tools/__init__.py M tests/__init__.py M tests/aspects.py A tests/pwb/print_env.py M tests/pwb/print_locals.py A tests/pwb/print_unicode.py M tests/pwb_tests.py M tests/utils.py 10 files changed, 151 insertions(+), 60 deletions(-)

Approvals: John Vandenberg: Looks good to me, but someone else must approve XZise: Looks good to me, approved jenkins-bot: Verified

diff --git a/pwb.py b/pwb.py index 365a22e..b1bb52c 100644 --- a/pwb.py +++ b/pwb.py @@ -79,7 +79,8 @@ sys.path[0] = os.path.dirname(filename)

try: - source = open(filename).read() + with open(filename, 'rb') as f: + source = f.read() exec(compile(source, filename, "exec", dont_inherit=True), main_mod.__dict__) finally: @@ -92,6 +93,16 @@ pwb.argvu = old_argvu

# end of snippet from coverage + + +def abspath(path): + """Convert path to absolute path, with uppercase drive letter on win32.""" + path = os.path.abspath(path) + if path[0] != '/': + # normalise Windows drive letter + path = path[0].upper() + path[1:] + return path +

if sys.version_info[0] not in (2, 3): raise RuntimeError("ERROR: Pywikibot only runs under Python 2 " @@ -107,9 +118,7 @@ # Establish a normalised path for the directory containing pwb.py. # Either it is '.' if the user's current working directory is the same, # or it is the absolute path for the directory of pwb.py -absolute_path = os.path.dirname(sys.argv[0]) -if not os.path.isabs(absolute_path): - absolute_path = os.path.abspath(os.path.join(os.curdir, absolute_path)) +absolute_path = abspath(os.path.dirname(sys.argv[0])) rewrite_path = absolute_path

sys.path = [sys.path[0], rewrite_path, @@ -157,7 +166,10 @@ # If successful, user-config.py already exists in one of the candidate # directories. See config2.py for details on search order. # Use env var to communicate to config2.py pwb.py location (bug 72918). - os.environ['PYWIKIBOT2_DIR_PWB'] = os.path.split(__file__)[0] + _pwb_dir = os.path.split(__file__)[0] + if sys.platform == 'win32' and sys.version_info[0] < 3: + _pwb_dir = str(_pwb_dir) + os.environ[str('PYWIKIBOT2_DIR_PWB')] = _pwb_dir import pywikibot # noqa except RuntimeError as err: # user-config.py to be created @@ -195,10 +207,10 @@ # This is a rough (and quick!) emulation of 'package name' detection. # a much more detailed implementation is in coverage's find_module. # https://bitbucket.org/ned/coveragepy/src/default/coverage/execfile.py - cwd = os.path.abspath(os.getcwd()) + cwd = abspath(os.getcwd()) if absolute_path == cwd: - absolute_filename = os.path.abspath(filename) - if absolute_filename.startswith(rewrite_path): + absolute_filename = abspath(filename)[:len(cwd)] + if absolute_filename == cwd: relative_filename = os.path.relpath(filename) # remove the filename, and use '.' instead of path separator. file_package = os.path.dirname( diff --git a/pywikibot/config2.py b/pywikibot/config2.py index db23c5e..730d4dd 100644 --- a/pywikibot/config2.py +++ b/pywikibot/config2.py @@ -46,8 +46,6 @@

from warnings import warn

-from pywikibot.tools import default_encoding - # This frozen set should contain all imported modules/variables, so it must # occur directly after the imports. At that point globals() only contains the # names and some magic variables (like __name__) @@ -346,7 +344,10 @@ # This default code should work fine, so you don't have to think about it. # TODO: consider getting rid of this config variable. try: - console_encoding = sys.stdout.encoding + if sys.version_info[0] > 2 or not sys.stdout.encoding: + console_encoding = sys.stdout.encoding + else: + console_encoding = sys.stdout.encoding.decode('ascii') except: # When using pywikibot inside a daemonized twisted application, # we get "StdioOnnaStick instance has no attribute 'encoding'" @@ -933,7 +934,11 @@ _ConfigurationDeprecationWarning)

# Fix up default console_encoding -console_encoding = default_encoding(console_encoding) +if console_encoding is None: + if sys.platform == 'win32': + console_encoding = 'cp850' + else: + console_encoding = 'iso-8859-1'

# Fix up transliteration_target if transliteration_target == 'not set': diff --git a/pywikibot/tools/__init__.py b/pywikibot/tools/__init__.py index 67e0087..22d6c4d 100644 --- a/pywikibot/tools/__init__.py +++ b/pywikibot/tools/__init__.py @@ -405,26 +405,6 @@ self.stop()

-def stream_encoding(stream): - """Get encoding of the stream and use a default if not existent/None.""" - try: - encoding = stream.encoding - except AttributeError: - encoding = None - return default_encoding(encoding) - - -def default_encoding(encoding): - """Return an encoding even if it's originally None.""" - if encoding is None: - if sys.platform == 'win32': - return 'cp850' - else: - return 'iso-8859-1' - else: - return encoding - - def itergroup(iterable, size): """Make an iterator that returns lists of (up to) size items from iterable.

diff --git a/tests/__init__.py b/tests/__init__.py index 031b90d..3f213d2 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -191,7 +191,8 @@ # overridden here to restrict retries to only 1, so developer builds fail more # frequently in code paths resulting from mishandled server problems. if config.max_retries > 2: - print('max_retries reduced from %d to 1 for tests' % config.max_retries) + if 'PYWIKIBOT_TEST_QUIET' not in os.environ: + print('tests: max_retries reduced from %d to 1' % config.max_retries) config.max_retries = 1

cache_misses = 0 diff --git a/tests/aspects.py b/tests/aspects.py index 78cd606..2458b6b 100644 --- a/tests/aspects.py +++ b/tests/aspects.py @@ -1157,14 +1157,17 @@ self.orig_pywikibot_dir = None if 'PYWIKIBOT2_DIR' in os.environ: self.orig_pywikibot_dir = os.environ['PYWIKIBOT2_DIR'] - os.environ['PYWIKIBOT2_DIR'] = pywikibot.config.base_dir + base_dir = pywikibot.config.base_dir + if sys.platform == 'win32' and sys.version_info[0] < 3: + base_dir = str(base_dir) + os.environ[str('PYWIKIBOT2_DIR')] = base_dir

def tearDown(self): """Restore the environment after running the pwb.py script.""" super(PwbTestCase, self).tearDown() del os.environ['PYWIKIBOT2_DIR'] if self.orig_pywikibot_dir: - os.environ['PYWIKIBOT2_DIR'] = self.orig_pywikibot_dir + os.environ[str('PYWIKIBOT2_DIR')] = self.orig_pywikibot_dir

def _execute(self, args, data_in=None, timeout=0, error=None): from tests.utils import execute_pwb diff --git a/tests/pwb/print_env.py b/tests/pwb/print_env.py new file mode 100644 index 0000000..e61230d --- /dev/null +++ b/tests/pwb/print_env.py @@ -0,0 +1,31 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +"""Script that forms part of pwb_tests.""" +from __future__ import unicode_literals + +import os +import sys + +_pwb_dir = os.path.abspath(os.path.join( + os.path.split(__file__)[0], '..', '..')) +_pwb_dir = _pwb_dir[0].upper() + _pwb_dir[1:] + +print('os.environ:') +for k, v in sorted(os.environ.items()): + # Don't leak the password into logs + if k == 'USER_PASSWORD': + continue + # This only appears in subprocesses + if k in ['PYWIKIBOT2_DIR_PWB']: + continue + print("%r: %r" % (k, v)) + +print('sys.path:') +for path in sys.path: + if path == '' or path.startswith('.'): + continue + # Normalise DOS drive letter + path = path[0].upper() + path[1:] + if path.startswith(_pwb_dir): + continue + print(path) diff --git a/tests/pwb/print_locals.py b/tests/pwb/print_locals.py index c39b776..261327a 100644 --- a/tests/pwb/print_locals.py +++ b/tests/pwb/print_locals.py @@ -1,3 +1,5 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- """Script that forms part of pwb_tests.""" from __future__ import unicode_literals

diff --git a/tests/pwb/print_unicode.py b/tests/pwb/print_unicode.py new file mode 100644 index 0000000..a8fa757 --- /dev/null +++ b/tests/pwb/print_unicode.py @@ -0,0 +1,9 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +"""Script that forms part of pwb_tests.""" +from __future__ import unicode_literals + +import pywikibot + +pywikibot.output('Häuser') +print('Häuser') diff --git a/tests/pwb_tests.py b/tests/pwb_tests.py index fe52837..80bcf87 100644 --- a/tests/pwb_tests.py +++ b/tests/pwb_tests.py @@ -22,9 +22,7 @@ from tests.utils import execute, execute_pwb from tests.aspects import unittest, PwbTestCase

-testbasepath = os.path.join(_tests_dir, 'pwb') -print_locals_test_package = 'tests.pwb.print_locals' -print_locals_test_script = os.path.join(testbasepath, 'print_locals.py') +_pwb_tests_dir = os.path.join(_tests_dir, 'pwb')

class TestPwb(PwbTestCase): @@ -42,17 +40,44 @@ site = False net = False

- def testScriptEnvironment(self): + def _do_check(self, name): + package_name = 'tests.pwb.' + name + script_path = os.path.join(_pwb_tests_dir, name + '.py') + + direct = execute([sys.executable, '-m', package_name]) + vpwb = execute_pwb([script_path]) + self.maxDiff = None + self.assertEqual(direct['stdout'], vpwb['stdout']) + + return (direct, vpwb) + + def test_env(self): """ - Test environment of pywikibot. + Test external environment of pywikibot.

Make sure the environment is not contaminated, and is the same as the environment we get when directly running a script. """ - direct = execute([sys.executable, '-m', 'tests.pwb.print_locals']) - vpwb = execute_pwb([print_locals_test_script]) - self.maxDiff = None - self.assertEqual(direct['stdout'], vpwb['stdout']) + self._do_check('print_env') + + def test_locals(self): + """ + Test internal environment of pywikibot. + + Make sure the environment is not contaminated, and is the same as + the environment we get when directly running a script. + """ + self._do_check('print_locals') + + def test_unicode(self): + """Test printing unicode in pywikibot.""" + (direct, vpwb) = self._do_check('print_unicode') + + self.assertEqual('Häuser', direct['stdout'].strip()) + self.assertEqual('Häuser', direct['stderr'].strip()) + self.assertEqual('Häuser', vpwb['stdout'].strip()) + self.assertEqual('Häuser', vpwb['stderr'].strip()) +

if __name__ == "__main__": unittest.main(verbosity=10) diff --git a/tests/utils.py b/tests/utils.py index d23fd91..9833916 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -16,7 +16,9 @@ from warnings import warn

import pywikibot -from pywikibot.tools import SelfCallDict, stream_encoding + +from pywikibot import config +from pywikibot.tools import SelfCallDict from pywikibot.site import Namespace from pywikibot.data.api import CachedRequest from pywikibot.data.api import Request as _original_Request @@ -259,16 +261,24 @@ # Any environment variables added on Windows must be of type # str() on Python 2. env = os.environ.copy() + + # Prevent output by test package; e.g. 'max_retries reduced from x to y' + env[str('PYWIKIBOT_TEST_QUIET')] = str('1') + # sys.path may have been modified by the test runner to load dependencies. - env['PYTHONPATH'] = ":".join(sys.path) + pythonpath = os.pathsep.join(sys.path) + if sys.platform == 'win32' and sys.version_info[0] < 3: + pythonpath = str(pythonpath) + env[str('PYTHONPATH')] = pythonpath + env[str('PYTHONIOENCODING')] = str(config.console_encoding) + # LC_ALL is used by i18n.input as an alternative for userinterface_lang if pywikibot.config.userinterface_lang: - env['LC_ALL'] = str(pywikibot.config.userinterface_lang) + env[str('LC_ALL')] = str(pywikibot.config.userinterface_lang) + # Set EDITOR to an executable that ignores all arguments and does nothing. - if sys.platform == 'win32': - env['EDITOR'] = str('call') - else: - env['EDITOR'] = 'true' + env[str('EDITOR')] = str('call' if sys.platform == 'win32' else 'true') + options = { 'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE @@ -276,14 +286,27 @@ if data_in is not None: options['stdin'] = subprocess.PIPE

- p = subprocess.Popen(command, env=env, **options) - - stdin_encoding = stream_encoding(p.stdin) - stdout_encoding = stream_encoding(p.stdout) - stderr_encoding = stream_encoding(p.stderr) + try: + p = subprocess.Popen(command, env=env, **options) + except TypeError: + # Generate a more informative error + if sys.platform == 'win32' and sys.version_info[0] < 3: + unicode_env = [(k, v) for k, v in os.environ.items() + if not isinstance(k, str) or + not isinstance(v, str)] + if unicode_env: + raise TypeError('os.environ must contain only str: %r' + % unicode_env) + child_unicode_env = [(k, v) for k, v in env.items() + if not isinstance(k, str) or + not isinstance(v, str)] + if child_unicode_env: + raise TypeError('os.environ must contain only str: %r' + % child_unicode_env) + raise

if data_in is not None: - p.stdin.write(data_in.encode(stdin_encoding)) + p.stdin.write(data_in.encode(config.console_encoding)) p.stdin.flush() # _communicate() otherwise has a broken pipe

stderr_lines = b'' @@ -296,7 +319,7 @@ if error: line = p.stderr.readline() stderr_lines += line - if error in line.decode(stdout_encoding): + if error in line.decode(config.console_encoding): break time.sleep(1) waited += 1 @@ -309,8 +332,8 @@

data_out = p.communicate() return {'exit_code': p.returncode, - 'stdout': data_out[0].decode(stdout_encoding), - 'stderr': (stderr_lines + data_out[1]).decode(stderr_encoding)} + 'stdout': data_out[0].decode(config.console_encoding), + 'stderr': (stderr_lines + data_out[1]).decode(config.console_encoding)}

def execute_pwb(args, data_in=None, timeout=0, error=None):