jenkins-bot has submitted this change and it was merged.
Change subject: Use load_tests protocol within test suite ......................................................................
Use load_tests protocol within test suite
Allow the list of tests to be specified for both Python 2.6 and 2.7 setup.py using a collector method in __init__.py, and use load_tests so nosetests also uses the same list of tests to be run by default.
Provide an ordering that tests basic components first, and steps up to test scripts which exercise larger parts of the package together.
Disable the ui and weblib tests in the module tests collector instead of within those test files. This will allow the ui test module to be restructured so that the functionality is visible when not called on the command line.
Allow individual tests to be disabled to assist with identifying which tests are causing problems.
Revise script tests so that test functions exist for scripts which are problematic, but those tests not included in the default list.
Change-Id: Ic0e18457888b428222a9dcbefcec9e1740b573aa --- M setup.py M tests/__init__.py M tests/script_tests.py M tests/utils.py M tox.ini 5 files changed, 230 insertions(+), 42 deletions(-)
Approvals: John Vandenberg: Looks good to me, but someone else must approve Merlijn van Deen: Looks good to me, approved jenkins-bot: Verified
diff --git a/setup.py b/setup.py index 4d64d1f..422561e 100644 --- a/setup.py +++ b/setup.py @@ -15,7 +15,6 @@ import itertools
test_deps = [] -testcollector = "tests"
dependencies = ['httplib2>=0.6.0']
@@ -40,7 +39,6 @@ sys.modules['unittest'] = unittest2
script_deps['replicate_wiki.py'] = ['argparse'] - testcollector = "tests.utils.collector" dependencies.append('ordereddict')
if sys.version_info[0] == 3: @@ -108,7 +106,7 @@ ], url='https://www.mediawiki.org/wiki/Pywikibot', download_url='https://github.com/wikimedia/pywikibot-core/archive/master.zip#egg=pywikibot...', - test_suite=testcollector, + test_suite="tests.collector", tests_require=test_deps, classifiers=[ 'License :: OSI Approved :: MIT License', diff --git a/tests/__init__.py b/tests/__init__.py index 919064a..96c3f95 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -36,11 +36,137 @@ "Python 2.6") sys.exit(1)
+if sys.version_info < (2, 7): + # Unittest2 is a backport of python 2.7s unittest module to python 2.6 + import unittest2 as unittest +else: + import unittest + import pywikibot.data.api from pywikibot.data.api import Request as _original_Request from pywikibot.data.api import CachedRequest
-_cache_dir = os.path.join(os.path.split(__file__)[0], 'apicache') +_tests_dir = os.path.split(__file__)[0] +_cache_dir = os.path.join(_tests_dir, 'apicache') + +library_test_modules = [ + 'date', + 'ipregex', + 'xmlreader', + 'textlib', + 'http', + 'namespace', + 'dry_api', + 'dry_site', + 'api', + 'site', + 'page', + 'file', + 'timestripper', + 'pagegenerators', + 'wikidataquery', + 'weblib', + 'i18n', + 'ui', + 'wikibase', +] + +script_test_modules = [ + 'pwb', + 'script', + 'archivebot', +] + +disabled_test_modules = [ + 'ui', +] + +if os.environ.get('TRAVIS', 'false') == 'true': + disabled_test_modules.append('weblib') + +disabled_tests = { + 'textlib': [ + 'test_interwiki_format', # example; very slow test + ] +} + + +def _unknown_test_modules(): + """List tests which are to be executed.""" + dir_list = os.listdir(_tests_dir) + all_test_list = [name[0:-9] for name in dir_list # strip '_tests.py' + if name.endswith('_tests.py') + and not name.startswith('_')] # skip __init__.py and _* + + unknown_test_modules = [name + for name in all_test_list + if name not in library_test_modules + and name not in script_test_modules] + + return unknown_test_modules + + +extra_test_modules = sorted(_unknown_test_modules()) + +test_modules = library_test_modules + extra_test_modules + script_test_modules + + +def collector(loader=unittest.loader.defaultTestLoader): + """Load the default modules. + + This is the entry point is specified in setup.py + """ + # Note: Raising SkipTest during load_tests will + # cause the loader to fallback to its own + # discover() ordering of unit tests. + if disabled_test_modules: + print('Disabled test modules (to run: python -m unittest ...):\n %s' + % ', '.join(disabled_test_modules)) + + if extra_test_modules: + print('Extra test modules (run after library, before scripts):\n %s' + % ', '.join(extra_test_modules)) + + if disabled_tests: + print('Skipping tests (to run: python -m unittest ...):\n %r' + % disabled_tests) + + modules = [module + for module in library_test_modules + + extra_test_modules + + script_test_modules + if module not in disabled_test_modules] + + test_list = [] + + for module in modules: + module_class_name = 'tests.' + module + '_tests' + if module in disabled_tests: + discovered = loader.loadTestsFromName(module_class_name) + enabled_tests = [] + for cls in discovered: + for test_func in cls: + if test_func._testMethodName not in disabled_tests[module]: + enabled_tests.append( + module_class_name + '.' + + test_func.__class__.__name__ + '.' + + test_func._testMethodName) + + test_list.extend(enabled_tests) + else: + test_list.append(module_class_name) + + tests = loader.loadTestsFromNames(test_list) + suite = unittest.TestSuite() + suite.addTests(tests) + return suite + + +def load_tests(loader=unittest.loader.defaultTestLoader, + tests=None, pattern=None): + """Load the default modules.""" + return collector(loader) +
CachedRequest._get_cache_dir = staticmethod( lambda *args: CachedRequest._make_dir(_cache_dir)) diff --git a/tests/script_tests.py b/tests/script_tests.py index 27a2ce3..32073de 100644 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -22,15 +22,15 @@
script_deps = { - 'script_wui.py': ['crontab', 'lua'], + 'script_wui': ['crontab', 'lua'], # Note: package 'lunatic-python' provides module 'lua'
- 'flickrripper.py': ['ImageTk', 'flickrapi'], + 'flickrripper': ['ImageTk', 'flickrapi'], # Note: 'PIL' is not available via pip2.7 on MS Windows, # however it is available with setuptools. } if sys.version_info < (2, 7): - script_deps['replicate_wiki.py'] = ['argparse'] + script_deps['replicate_wiki'] = ['argparse']
def check_script_deps(script_name): @@ -46,21 +46,31 @@ return True
-def runnable_script_list(scripts_path): - """List of scripts which may be executed.""" - dir_list = os.listdir(scripts_path) - script_list = [name[0:-3] for name in dir_list # strip .py - if name.endswith('.py') - and not name.startswith('_') # skip __init__.py and _* - and check_script_deps(name) - and name != 'login.py' # this is moved to be first - and name != 'imageuncat.py' # this halts indefinitely - and name != 'welcome.py' # result depends on speed - and name != 'script_wui.py' # depends on lua compiling - and name != 'editarticle.py' # requires a X DISPLAY - and name != 'makecat.py' # bug 69781 - ] - return ['login'] + script_list +failed_dep_script_list = [name + for name in script_deps + if not check_script_deps(name)] + +unrunnable_script_list = [ + 'script_wui', # depends on lua compiling + 'editarticle', # requires a X DISPLAY +] + +deadlock_script_list = [ + 'makecat', # bug 69781 +] + +script_list = (['login'] + + [name[0:-3] for name in os.listdir(scripts_path) # strip '.py' + if name.endswith('.py') + and not name.startswith('_') # skip __init__.py and _* + and name != 'login.py' # this is moved to be first + ] + ) + +runnable_script_list = (['login'] + + sorted(set(script_list) - + set(['login']) - + set(unrunnable_script_list)))
script_input = { 'catall': 'q\n', # q for quit @@ -85,6 +95,7 @@ 'clean_sandbox', 'disambredir', 'imagerecat', + 'login', 'lonelypages', 'misspelling', 'revertbot', @@ -130,6 +141,45 @@ 'revertbot': 'Fetching new batch of contributions', 'upload': 'ERROR: Upload error', } + + +def collector(loader=unittest.loader.defaultTestLoader): + """Load the default tests.""" + # Note: Raising SkipTest during load_tests will + # cause the loader to fallback to its own + # discover() ordering of unit tests. + + enable_autorun_tests = ( + os.environ.get('PYWIKIBOT2_TEST_AUTORUN', '0') == '1') + + tests = (['test__login_execution'] + + ['test_' + name + '_execution' + for name in sorted(script_list) + if name != 'login' + and name not in deadlock_script_list] + + ['test__login_no_args']) + + tests += ['test_' + name + '_no_args' + for name in sorted(script_list) + if name != 'login' + and name not in deadlock_script_list + and name not in failed_dep_script_list # no_args = execution + and name not in unrunnable_script_list + and (enable_autorun_tests or name not in auto_run_script_list)] + + test_list = ['tests.script_tests.TestScript.' + name + for name in tests] + + tests = loader.loadTestsFromNames(test_list) + suite = unittest.TestSuite() + suite.addTests(tests) + return suite + + +def load_tests(loader=unittest.loader.defaultTestLoader, + tests=None, pattern=None): + """Load the default modules.""" + return collector(loader)
def execute(command, data_in=None, timeout=0): @@ -195,10 +245,12 @@ result['stderr']) return testScript
- for script_name in runnable_script_list(scripts_path): + for script_name in script_list: # force login to be the first, alphabetically, so the login # message does not unexpectedly occur during execution of # another script. + # unrunnable script tests are disabled by default in load_tests() + if script_name == 'login': test_name = 'test__' + script_name + '_execution' else: @@ -212,6 +264,17 @@ ]: dct[test_name] = unittest.expectedFailure(dct[test_name]) dct[test_name].__doc__ = 'Test running ' + script_name + '.' + dct[test_name].__name__ = test_name + + # Ideally all scripts should execute -help without + # connecting to a site. However pywikibot always + # logs site.live_version(). + # TODO: make logging live_version() optional, then set + # dct[test_name].site = True + # for only the tests which dont respond to -help + + if script_name in deadlock_script_list: + dct[test_name].__test__ = False
if script_name == 'login': test_name = 'test__' + script_name + '_no_args' @@ -222,6 +285,7 @@ if script_name in ['checkimages', # bug 68613 'data_ingestion', # bug 68611 'flickrripper', # bug 68606 (and deps) + 'script_wui', # Error on any user except DrTrigonBot 'upload', # raises custom ValueError ] or ( ((config.family != 'wikipedia' or config.mylang != 'en') and script_name == 'cfd') or @@ -232,12 +296,28 @@ 'Test running ' + script_name + ' without arguments.' dct[test_name].__name__ = test_name
+ # Disable test bt default in nosetests + if script_name in unrunnable_script_list + deadlock_script_list: + dct[test_name].__test__ = False + + # TODO: Ideally any script not on the auto_run_script_list + # can be set as 'not a site' test, but that will require + # auditing all code in main() to ensure it exits without + # connecting to a site. There are outstanding bugs about + # connections during initialisation. + # + # dct[test_name].site = True + return type.__new__(cls, name, bases, dct)
class TestScript(PywikibotTestCase):
- """Test cases for scripts.""" + """Test cases for scripts. + + This class sets the nose 'site' attribute on each test + depending on whether it is in the auto_run_script_list. + """
__metaclass__ = TestScriptMeta
diff --git a/tests/utils.py b/tests/utils.py index ef1a15f..07b46c8 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -9,27 +9,11 @@ # import time import sys - -if sys.version_info < (2, 7): - # Unittest2 is a backport of python 2.7s unittest module to python 2.6 - import unittest2 as unittest -else: - import unittest - import pywikibot +from tests import patch_request, unpatch_request, unittest
# Number of seconds each test may consume before a note is added after the test. test_duration_warning_interval = 10 - - -def collector(): - # This test collector loads tests from the `tests` directory in files - # matching the pattern `*tests.py`. This gets used by `setup.py test` when - # running on Python 2.6 to use the unittest2 test runner instead of the - # unittest one. - return unittest.loader.defaultTestLoader.discover("tests", "*tests.py") - -from tests import patch_request, unpatch_request
class BaseTestCase(unittest.TestCase): diff --git a/tox.ini b/tox.ini index 9339bcc..1725c50 100644 --- a/tox.ini +++ b/tox.ini @@ -19,7 +19,7 @@
[testenv:nose] setenv = PYWIKIBOT2_NO_USER_CONFIG=1 -commands = nosetests -a "!site,!net" +commands = nosetests -a "!site,!net" tests deps = nose
[testenv:venv]