jenkins-bot has submitted this change and it was merged.
Change subject: Miscellaneous pwb improvements ......................................................................
Miscellaneous pwb improvements
Improve pwb emulation of Python 3, so that it sets __builtins__ and __package__, doesnt use imp module deprecated in Python 3.4. These changes are partial sync with the coverage code from whence it came.
Minor updates to our pwb test so it now works in Python 3. The same coverage test is now wildly more complex; beyond pywikibot's needs.
c5d5b01 removed Site interaction from the library bootstrap logic. The pwb tests can now be flagged as 'net' only, and not 'site', tests. The tests now run pre-merge with jenkins.
Move the pwb running functionality out of script_tests into utils.
Bug: T72336 Change-Id: Ic2443c17c9ad3976d654b2dd29201cfc12134322 --- M pwb.py M tests/__init__.py M tests/aspects.py A tests/pwb/__init__.py M tests/pwb/print_locals.py M tests/pwb_tests.py M tests/script_tests.py M tests/utils.py 8 files changed, 235 insertions(+), 161 deletions(-)
Approvals: John Vandenberg: Looks good to me, but someone else must approve Merlijn van Deen: Looks good to me, approved jenkins-bot: Verified
diff --git a/pwb.py b/pwb.py index 13614e6..ec6840a 100644 --- a/pwb.py +++ b/pwb.py @@ -12,16 +12,19 @@ # # Distributed under the terms of the MIT license. # +from __future__ import print_function __version__ = '$Id$'
# The following snippet was developed by Ned Batchelder (and others) -# for coverage.py [1], and is available under the BSD license (see [2]) +# for coverage [1], with python 3 support [2] added later, +# and is available under the BSD license (see [3]) # [1] https://bitbucket.org/ned/coveragepy/src/b5abcee50dbe/coverage/execfile.py -# [2] https://bitbucket.org/ned/coveragepy/src/2c5fb3a8b81cc56d8ad57dd1bd83ef7740f... +# [2] https://bitbucket.org/ned/coveragepy/src/fd5363090034/coverage/execfile.py +# [3] https://bitbucket.org/ned/coveragepy/src/2c5fb3a8b81c/setup.py?at=default#cl...
-import imp import os import sys +import types
pwb = None
@@ -42,7 +45,7 @@ pwb.argvu = []
-def run_python_file(filename, argv, argvu): +def run_python_file(filename, argv, argvu, package=None): """Run a python file as if it were the main program on the command line.
`filename` is the path to the file to execute, it need not be a .py file. @@ -53,13 +56,15 @@
# Create a module to serve as __main__ old_main_mod = sys.modules['__main__'] - main_mod = imp.new_module('__main__') + main_mod = types.ModuleType('__main__') sys.modules['__main__'] = main_mod main_mod.__file__ = filename if sys.version_info[0] > 2: - main_mod.builtins = sys.modules['builtins'] + main_mod.__builtins__ = sys.modules['builtins'] else: main_mod.__builtins__ = sys.modules['__builtin__'] + if package: + main_mod.__package__ = package
# Set sys.argv and the first path element properly. old_argv = sys.argv @@ -95,9 +100,13 @@ raise RuntimeError("ERROR: Pywikibot only runs under Python 3.3 " "or higher")
-rewrite_path = os.path.dirname(sys.argv[0]) -if not os.path.isabs(rewrite_path): - rewrite_path = os.path.abspath(os.path.join(os.curdir, rewrite_path)) +# Establish a normalised path for the directory containing pwb.py. +# Either it is '.' if the user's current working directory is the same, +# or it is the absolute path for the directory of pwb.py +absolute_path = os.path.dirname(sys.argv[0]) +if not os.path.isabs(absolute_path): + absolute_path = os.path.abspath(os.path.join(os.curdir, absolute_path)) +rewrite_path = absolute_path
sys.path = [sys.path[0], rewrite_path, os.path.join(rewrite_path, 'pywikibot', 'compat'), @@ -165,19 +174,50 @@ []) sys.exit(1)
-if len(sys.argv) > 1: - tryimport_pwb() - fn = sys.argv[1] - argv = sys.argv[1:] - argvu = pwb.argvu[1:] - if not fn.endswith('.py'): - fn += '.py' - if not os.path.exists(fn): - testpath = os.path.join(os.path.split(__file__)[0], 'scripts', fn) - if os.path.exists(testpath): - fn = testpath - else: - raise OSError("%s not found!" % fn) - run_python_file(fn, argv, argvu) -elif __name__ == "__main__": - print(__doc__) +if __name__ == "__main__": + if len(sys.argv) > 1: + file_package = None + tryimport_pwb() + filename = sys.argv[1] + argv = sys.argv[1:] + argvu = pwb.argvu[1:] + if not filename.endswith('.py'): + filename += '.py' + if not os.path.exists(filename): + testpath = os.path.join(os.path.split(__file__)[0], + 'scripts', + filename) + file_package = 'scripts' + if os.path.exists(testpath): + filename = testpath + else: + raise OSError("%s not found!" % filename) + + # When both pwb.py and the filename to run are within the current + # working directory: + # a) set __package__ as if called using python -m scripts.blah.foo + # b) set __file__ to be relative, so it can be relative in backtraces, + # and __file__ *appears* to be an unstable path to load data from. + # This is a rough (and quick!) emulation of 'package name' detection. + # a much more detailed implementation is in coverage's find_module. + # https://bitbucket.org/ned/coveragepy/src/default/coverage/execfile.py + cwd = os.path.abspath(os.getcwd()) + if absolute_path == cwd: + absolute_filename = os.path.abspath(filename) + if absolute_filename.startswith(rewrite_path): + relative_filename = os.path.relpath(filename) + # remove the filename, and use '.' instead of path separator. + file_package = os.path.dirname( + relative_filename).replace(os.sep, '.') + filename = os.path.join(os.curdir, relative_filename) + + if file_package and file_package not in sys.modules: + try: + __import__(file_package) + except ImportError as e: + print('Parent module %s not found: %s' + % (file_package, e), file=sys.stderr) + + run_python_file(filename, argv, argvu, file_package) + else: + print(__doc__) diff --git a/tests/__init__.py b/tests/__init__.py index d32f9f1..2750203 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -52,6 +52,10 @@ _cache_dir = os.path.join(_tests_dir, 'apicache') _data_dir = os.path.join(_tests_dir, 'data')
+# Find the root directory of the checkout +_root_dir = os.path.split(_tests_dir)[0] +_pwb_py = os.path.join(_root_dir, 'pwb.py') + library_test_modules = [ 'deprecation', 'date', diff --git a/tests/aspects.py b/tests/aspects.py index 5e39c0c..f220d15 100644 --- a/tests/aspects.py +++ b/tests/aspects.py @@ -542,13 +542,6 @@ # % (base.__name__, key, name)) dct[key] = getattr(base, key)
- if 'pwb' in dct and dct['pwb']: - dct['spawn'] = True - if 'site' not in dct: - raise Exception( - '%s: Test classes using pwb must set "site"' - % name) - if 'net' in dct and dct['net'] is False: dct['site'] = False
@@ -577,6 +570,17 @@ ('site' in dct and not dct['site'])): # Prevent use of pywikibot.Site bases = tuple([DisableSiteMixin] + list(bases)) + + # 'pwb' tests will _usually_ require a site. To ensure the + # test class dependencies are declarative, this requires the + # test writer explicitly sets 'site=False' so code reviewers + # check that the script invoked by pwb will not load a site. + if 'pwb' in dct and dct['pwb']: + if 'site' not in dct: + raise Exception( + '%s: Test classes using pwb must set "site"; add ' + 'site=False if the test script will not use a site' + % name)
# If the 'site' attribute is a false value, # remove it so it matches !site in nose. @@ -939,15 +943,38 @@
class PwbTestCase(TestCase):
- """Test cases use pwb.py to invoke scripts.""" + """ + Test cases use pwb.py to invoke scripts. + + Test cases which use pwb typically also access a site, and use the network. + Even during initialisation, scripts may call pywikibot.handle_args, which + initialises loggers and uses the network to determine if the code is stale. + + The flag 'pwb' is used by the TestCase metaclass to check that a test site + is set declared in the class properties, or that 'site = False' is added + to the class properties in the unlikely scenario that the test case + uses pwb in a way that doesnt use a site. + + If a test class is marked as 'site = False', the metaclass will also check + that the 'net' flag is explicitly set. + """
pwb = True - spawn = True - # pywikibot.handleArgs currently instantiates a Site object - # and tries to fetch the users messages. - site = True - net = True - user = True + + def setUp(self): + """Prepare the environment for running the pwb.py script.""" + super(PwbTestCase, self).setUp() + self.orig_pywikibot_dir = None + if 'PYWIKIBOT2_DIR' in os.environ: + self.orig_pywikibot_dir = os.environ['PYWIKIBOT2_DIR'] + os.environ['PYWIKIBOT2_DIR'] = pywikibot.config.base_dir + + def tearDown(self): + """Restore the environment after running the pwb.py script.""" + super(PwbTestCase, self).tearDown() + del os.environ['PYWIKIBOT2_DIR'] + if self.orig_pywikibot_dir: + os.environ['PYWIKIBOT2_DIR'] = self.orig_pywikibot_dir
class DebugOnlyTestCase(TestCase): diff --git a/tests/pwb/__init__.py b/tests/pwb/__init__.py new file mode 100644 index 0000000..fb493b3 --- /dev/null +++ b/tests/pwb/__init__.py @@ -0,0 +1 @@ +"""Dummy package initialisation.""" diff --git a/tests/pwb/print_locals.py b/tests/pwb/print_locals.py index 3f94864..42749df 100644 --- a/tests/pwb/print_locals.py +++ b/tests/pwb/print_locals.py @@ -1,4 +1,12 @@ """Script that forms part of pwb_tests."""
-for k, v in locals().copy().items(): - print("%r: %r" % (k, v)) +import os.path + +for k, v in sorted(locals().copy().items()): + # Skip a few items that Python 3 adds and are not emulated in pwb. + if k in ['__cached__', '__loader__', '__spec__']: + continue + if k == '__file__': + print("__file__: %r" % os.path.join('.', os.path.relpath(__file__))) + else: + print("%r: %r" % (k, v)) diff --git a/tests/pwb_tests.py b/tests/pwb_tests.py index 883d4f4..e4c0285 100644 --- a/tests/pwb_tests.py +++ b/tests/pwb_tests.py @@ -1,5 +1,11 @@ # -*- coding: utf-8 -*- -"""Test pwb.py.""" +""" +Test pwb.py. + +If pwb.py does not load python files as expected, more tests from coverage +should be added locally. +https://bitbucket.org/ned/coveragepy/src/default/tests/test_execfile.py +""" # # (C) Pywikibot team, 2007-2014 # @@ -9,24 +15,17 @@
import os import sys -import subprocess -import pywikibot
-from tests.aspects import unittest, TestCase +from tests import _tests_dir +from tests.utils import execute, execute_pwb +from tests.aspects import unittest, PwbTestCase
-pypath = sys.executable -basepath = os.path.split(os.path.split(__file__)[0])[0] -pwbpath = os.path.join(basepath, 'pwb.py') -testbasepath = os.path.join(basepath, 'tests', 'pwb') +testbasepath = os.path.join(_tests_dir, 'pwb') +print_locals_test_package = 'tests.pwb.print_locals' +print_locals_test_script = os.path.join(testbasepath, 'print_locals.py')
-def check_output(command): - """Execute and return the output of a command.""" - return subprocess.Popen(command, stdout=subprocess.PIPE).communicate()[0] - - -@unittest.skipIf(sys.version_info[0] > 2, "The mapping is different in Python 3") -class TestPwb(TestCase): +class TestPwb(PwbTestCase):
""" Test pwb.py functionality. @@ -35,18 +34,11 @@ without a user-config.py """
- pwb = True - net = True - site = True - - def setUp(self): - self.oldenviron = os.environ.copy() - os.environ['PYWIKIBOT2_DIR'] = pywikibot.config.base_dir - - def tearDown(self): - del os.environ['PYWIKIBOT2_DIR'] - if 'PYWIKIBOT2_DIR' in self.oldenviron: - os.environ['PYWIKIBOT2_DIR'] = self.oldenviron['PYWIKIBOT2_DIR'] + # site must be explicitly set for pwb tests. This test does not require + # network access, because tests/pwb/print_locals.py does not use + # handle_args, etc. so version.py doesnt talk on the network. + site = False + net = False
def testScriptEnvironment(self): """ @@ -55,11 +47,10 @@ Make sure the environment is not contaminated, and is the same as the environment we get when directly running a script. """ - test = os.path.join(testbasepath, 'print_locals.py') - - direct = check_output([pypath, test]) - vpwb = check_output([pypath, pwbpath, test]) - self.assertEqual(direct, vpwb) + direct = execute([sys.executable, '-m', 'tests.pwb.print_locals']) + vpwb = execute_pwb([print_locals_test_script]) + self.maxDiff = None + self.assertEqual(direct['stdout'], vpwb['stdout'])
if __name__ == "__main__": unittest.main(verbosity=10) diff --git a/tests/script_tests.py b/tests/script_tests.py index f7e2422..8ee4be5 100644 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -10,18 +10,14 @@
import os import sys -import time -import subprocess
-import pywikibot from pywikibot import config
+from tests import _root_dir from tests.aspects import unittest, DefaultSiteTestCase, MetaTestCaseClass, PwbTestCase -from tests.utils import allowed_failure +from tests.utils import allowed_failure, execute_pwb
-base_path = os.path.split(os.path.split(__file__)[0])[0] -pwb_path = os.path.join(base_path, 'pwb.py') -scripts_path = os.path.join(base_path, 'scripts') +scripts_path = os.path.join(_root_dir, 'scripts')
script_deps = { 'script_wui': ['crontab', 'lua'], @@ -204,63 +200,6 @@ return collector(loader)
-def execute(command, data_in=None, timeout=0, error=None): - """Execute a command and capture outputs.""" - def decode(stream): - if sys.version_info[0] > 2: - return stream.decode(config.console_encoding) - else: - return stream - env = os.environ.copy() - # sys.path may have been modified by the test runner to load dependencies. - env['PYTHONPATH'] = ":".join(sys.path) - # Set EDITOR to an executable that ignores all arguments and does nothing. - if sys.platform == 'win32': - env['EDITOR'] = 'call' - else: - env['EDITOR'] = 'true' - options = { - 'stdout': subprocess.PIPE, - 'stderr': subprocess.PIPE - } - if data_in is not None: - options['stdin'] = subprocess.PIPE - - p = subprocess.Popen(command, env=env, **options) - - if data_in is not None: - if sys.version_info[0] > 2: - data_in = data_in.encode(config.console_encoding) - p.stdin.write(data_in) - p.stdin.flush() # _communicate() otherwise has a broken pipe - - stderr_lines = b'' - waited = 0 - while (error or (waited < timeout)) and p.poll() is None: - # In order to kill 'shell' and others early, read only a single - # line per second, and kill the process as soon as the expected - # output has been seen. - # Additional lines will be collected later with p.communicate() - if error: - line = p.stderr.readline() - stderr_lines += line - if error in decode(line): - break - time.sleep(1) - waited += 1 - - if (timeout or error) and p.poll() is None: - p.kill() - - if p.poll() is not None: - stderr_lines += p.stderr.read() - - data_out = p.communicate() - return {'exit_code': p.returncode, - 'stdout': decode(data_out[0]), - 'stderr': decode(stderr_lines + data_out[1])} - - class TestScriptMeta(MetaTestCaseClass):
"""Test meta class.""" @@ -269,7 +208,7 @@ """Create the new class.""" def test_execution(script_name, args=[], expected_results=None): def testScript(self): - cmd = [sys.executable, pwb_path, script_name] + cmd = [script_name]
if args: cmd += args @@ -285,7 +224,7 @@ else: error = None
- result = execute(cmd, data_in, timeout=timeout, error=error) + result = execute_pwb(cmd, data_in, timeout=timeout, error=error)
stderr = result['stderr'].split('\n') stderr_sleep = [l for l in stderr @@ -367,11 +306,9 @@ dct[test_name].__name__ = test_name
# Ideally all scripts should execute -help without - # connecting to a site. However pywikibot always - # logs site.version() from live wiki. - # TODO: make logging version() optional, then set - # dct[test_name].site = True - # for only the tests which dont respond to -help + # connecting to a site. + # TODO: after bug 68611 and 68664 (and makecat), split -help + # execution to a separate test class which uses site=False.
if script_name in deadlock_script_list: dct[test_name].__test__ = False @@ -420,26 +357,15 @@
"""Test cases for scripts.
- This class sets the nose 'site' attribute on each test - depending on whether it is in the auto_run_script_list. + This class sets the nose 'user' attribute on every test, thereby ensuring + that the test runner has a username for the default site, and so that + Site.login() is called in the test runner, which means that the scripts + run in pwb can automatically login using the saved cookies. """
__metaclass__ = TestScriptMeta
- def setUp(self): - """Prepare the environment for running the pwb.py script.""" - super(TestScript, self).setUp() - self.old_pywikibot_dir = None - if 'PYWIKIBOT2_DIR' in os.environ: - self.old_pywikibot_dir = os.environ['PYWIKIBOT2_DIR'] - os.environ['PYWIKIBOT2_DIR'] = pywikibot.config.base_dir - - def tearDown(self): - """Restore the environment after running the pwb.py script.""" - super(TestScript, self).tearDown() - del os.environ['PYWIKIBOT2_DIR'] - if self.old_pywikibot_dir: - os.environ['PYWIKIBOT2_DIR'] = self.old_pywikibot_dir + user = True
if sys.version_info[0] > 2: diff --git a/tests/utils.py b/tests/utils.py index 4f1b002..06661e0 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -8,6 +8,10 @@ from __future__ import print_function __version__ = '$Id$' # +import os +import subprocess +import sys +import time
import pywikibot from pywikibot.tools import SelfCallDict @@ -15,7 +19,7 @@ from pywikibot.data.api import CachedRequest from pywikibot.data.api import Request as _original_Request
-from tests import aspects +from tests import aspects, _pwb_py from tests import unittest # noqa
BaseTestCase = aspects.TestCase @@ -192,3 +196,76 @@ canonical_name='Property', defaultcontentmodel='wikibase-property') }) + + +def execute(command, data_in=None, timeout=0, error=None): + """ + Execute a command and capture outputs. + + @param command: executable to run and arguments to use + @type command: list of unicode + """ + def decode(stream): + if sys.version_info[0] > 2: + return stream.decode(pywikibot.config.console_encoding) + else: + return stream + env = os.environ.copy() + # sys.path may have been modified by the test runner to load dependencies. + env['PYTHONPATH'] = ":".join(sys.path) + # Set EDITOR to an executable that ignores all arguments and does nothing. + if sys.platform == 'win32': + env['EDITOR'] = 'call' + else: + env['EDITOR'] = 'true' + options = { + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE + } + if data_in is not None: + options['stdin'] = subprocess.PIPE + + p = subprocess.Popen(command, env=env, **options) + + if data_in is not None: + if sys.version_info[0] > 2: + data_in = data_in.encode(pywikibot.config.console_encoding) + p.stdin.write(data_in) + p.stdin.flush() # _communicate() otherwise has a broken pipe + + stderr_lines = b'' + waited = 0 + while (error or (waited < timeout)) and p.poll() is None: + # In order to kill 'shell' and others early, read only a single + # line per second, and kill the process as soon as the expected + # output has been seen. + # Additional lines will be collected later with p.communicate() + if error: + line = p.stderr.readline() + stderr_lines += line + if error in decode(line): + break + time.sleep(1) + waited += 1 + + if (timeout or error) and p.poll() is None: + p.kill() + + if p.poll() is not None: + stderr_lines += p.stderr.read() + + data_out = p.communicate() + return {'exit_code': p.returncode, + 'stdout': decode(data_out[0]), + 'stderr': decode(stderr_lines + data_out[1])} + + +def execute_pwb(args, data_in=None, timeout=0, error=None): + """ + Execute the pwb.py script and capture outputs. + + @param args: list of arguments for pwb.py + @type args: list of unicode + """ + return execute(command=[sys.executable, _pwb_py] + args, + data_in=data_in, timeout=timeout, error=error)