Pywikibot-commits August 2014

pywikibot-commits@lists.wikimedia.org

1 participants
196 discussions

[Gerrit] Fix cache key for non-Latin sites - change (pywikibot/core)

by Merlijn van Deen (Code Review)

Merlijn van Deen has submitted this change and it was merged. Change subject: Fix cache key for non-Latin sites ...................................................................... Fix cache key for non-Latin sites Avoiding using Page.__repr__() as it uses the localized namespace name and encodes the title into the console encoding, which causes problems when _create_file_name() tries to encode it again. Bug: 67488 Change-Id: If49039bac8e8347852c1a04963f678903621d2a9 --- M pywikibot/data/api.py M tests/dry_api_tests.py 2 files changed, 67 insertions(+), 19 deletions(-) Approvals: John Vandenberg: Looks good to me, but someone else must approve Phe: Looks good to me, but someone else must approve Merlijn van Deen: Looks good to me, approved diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index 33e46f6..7102074 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -483,14 +483,16 @@ if login_status > pywikibot.site.LoginStatus.NOT_LOGGED_IN and \ hasattr(self.site, '_userinfo') and \ 'name' in self.site._userinfo: - user_key = pywikibot.page.User(self.site, - self.site._userinfo['name']) + # This uses the format of Page.__repr__(), without the encoding + # it performs. This string cant be encoded otherwise it creates an + # exception when _create_file_name() tries to encode it again. + user_key = u'User(User:%s)' % self.site._userinfo['name'] else: user_key = pywikibot.site.LoginStatus( max(login_status, pywikibot.site.LoginStatus.NOT_LOGGED_IN)) + user_key = repr(user_key) - return (repr(self.site) + repr(user_key) + - repr(sorted(self.iteritems()))) + return repr(self.site) + user_key + repr(sorted(self.iteritems())) def _create_file_name(self): self.http_params() # normalize self.iteritems() diff --git a/tests/dry_api_tests.py b/tests/dry_api_tests.py index 95870ff..1714b59 100644 --- a/tests/dry_api_tests.py +++ b/tests/dry_api_tests.py @@ -8,12 +8,13 @@ # import datetime +import hashlib import pywikibot from pywikibot.data.api import CachedRequest, QueryGenerator from utils import unittest -class DryAPITests(unittest.TestCase): +class DryCachedRequestTests(unittest.TestCase): def setUp(self): self.basesite = pywikibot.Site('en', 'wikipedia') @@ -28,6 +29,10 @@ def test_expiry_formats(self): self.assertEqual(self.req.expiry, CachedRequest(datetime.timedelta(days=1), **self.parms).expiry) + + def test_expired(self): + self.assertFalse(self.req._expired(datetime.datetime.now())) + self.assertTrue(self.req._expired(datetime.datetime.now() - datetime.timedelta(days=2))) def test_get_cache_dir(self): retval = self.req._get_cache_dir() @@ -44,16 +49,27 @@ self.assertNotEqual(self.req._cachefile_path(), self.diffreq._cachefile_path()) self.assertNotEqual(self.req._cachefile_path(), self.diffsite._cachefile_path()) - def test_cachefile_path_different_users(self): - # Mock basesite object to test this. + +class MockCachedRequestKeyTests(unittest.TestCase): + def setUp(self): + class MockFamily(pywikibot.family.Family): + + @property + def name(self): + return 'mock' + class MockSite(pywikibot.site.APISite): _loginstatus = pywikibot.site.LoginStatus.NOT_ATTEMPTED - _namespaces = {2: 'User'} + _namespaces = {2: ['User']} def __init__(self): self._user = 'anon' + pywikibot.site.BaseSite.__init__(self, 'mock', MockFamily()) + + def languages(self): + return ['mock'] def user(self): return self._user @@ -74,33 +90,63 @@ def __getattr__(self, attr): raise Exception("Attribute %r not defined" % attr) - site = MockSite() - req = CachedRequest(expiry=1, site=site, action='query', meta='siteinfo') + self.site = MockSite() + + def test_cachefile_path_different_users(self): + req = CachedRequest(expiry=1, site=self.site, + action='query', meta='siteinfo') anonpath = req._cachefile_path() - site._userinfo = {'name': u'user'} - site._loginstatus = 0 - req = CachedRequest(expiry=1, site=site, action='query', meta='siteinfo') + self.site._userinfo = {'name': u'MyUser'} + self.site._loginstatus = 0 + req = CachedRequest(expiry=1, site=self.site, + action='query', meta='siteinfo') userpath = req._cachefile_path() self.assertNotEqual(anonpath, userpath) - site._userinfo = {'name': u'sysop'} - site._loginstatus = 1 - req = CachedRequest(expiry=1, site=site, action='query', meta='siteinfo') + self.site._userinfo = {'name': u'MySysop'} + self.site._loginstatus = 1 + req = CachedRequest(expiry=1, site=self.site, + action='query', meta='siteinfo') sysoppath = req._cachefile_path() self.assertNotEqual(anonpath, sysoppath) self.assertNotEqual(userpath, sysoppath) - def test_expired(self): - self.assertFalse(self.req._expired(datetime.datetime.now())) - self.assertTrue(self.req._expired(datetime.datetime.now() - datetime.timedelta(days=2))) + def test_unicode(self): + self.site._userinfo = {'name': u'محمد الفلسطيني'} + self.site._loginstatus = 0 + + req = CachedRequest(expiry=1, site=self.site, + action='query', meta='siteinfo') + en_user_path = req._cachefile_path() + + self.site._namespaces = {2: [u'مستخدم']} + + req = CachedRequest(expiry=1, site=self.site, + action='query', meta='siteinfo') + + expect = u'MockSite()User(User:محمد الفلسطيني)' + \ + "[('action', 'query'), ('meta', 'siteinfo')]" + + self.assertEqual(repr(req._uniquedescriptionstr()), repr(expect)) + + self.assertEqual(req._uniquedescriptionstr().encode('utf-8'), + expect.encode('utf-8')) + + ar_user_path = req._cachefile_path() + + self.assertEqual(en_user_path, ar_user_path) + + +class DryQueryGenTests(unittest.TestCase): def test_query_constructor(self): qGen1 = QueryGenerator(action="query", meta="siteinfo") qGen2 = QueryGenerator(meta="siteinfo") self.assertEqual(str(qGen1.request), str(qGen2.request)) + if __name__ == '__main__': unittest.main() -- To view, visit https://gerrit.wikimedia.org/r/149898 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: If49039bac8e8347852c1a04963f678903621d2a9 Gerrit-PatchSet: 4 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Phe <phil.el(a)free.fr> Gerrit-Reviewer: jenkins-bot <>

9 years, 8 months

[Gerrit] Test harness for scripts should login first - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Test harness for scripts should login first ...................................................................... Test harness for scripts should login first If the login does not happen first, the first script that attempts to login will unexpectedly emit Logging in to <site> as <user> Change-Id: I41111456149aab7da02613e581e692d4e1058b31 --- M tests/script_tests.py 1 file changed, 13 insertions(+), 3 deletions(-) Approvals: John Vandenberg: Looks good to me, but someone else must approve Merlijn van Deen: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/script_tests.py b/tests/script_tests.py index b371904..ef151ed 100644 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -53,12 +53,13 @@ if name.endswith('.py') and not name.startswith('_') # skip __init__.py and _* and check_script_deps(name) + and name != 'login.py' # this is moved to be first and name != 'imageuncat.py' # this halts indefinitely and name != 'watchlist.py' # result depends on speed and name != 'welcome.py' # result depends on speed and name != 'script_wui.py' # depends on lua compiling ] - return script_list + return ['login'] + script_list script_input = { 'catall': 'q\n', # q for quit @@ -192,7 +193,13 @@ return testScript for script_name in runnable_script_list(scripts_path): - test_name = 'test_' + script_name + '_execution' + # force login to be the first, alphabetically, so the login + # message does not unexpectedly occur during execution of + # another script. + if script_name == 'login': + test_name = 'test__' + script_name + '_execution' + else: + test_name = 'test_' + script_name + '_execution' dct[test_name] = test_execution(script_name, ['-help']) if script_name in ['shell', 'version', 'checkimages', # bug 68613 @@ -204,7 +211,10 @@ dct[test_name] = unittest.expectedFailure(dct[test_name]) dct[test_name].__doc__ = 'Test running ' + script_name + '.' - test_name = 'test_' + script_name + '_no_args' + if script_name == 'login': + test_name = 'test__' + script_name + '_no_args' + else: + test_name = 'test_' + script_name + '_no_args' dct[test_name] = test_execution(script_name, ['-simulate'], no_args_expected_results) if script_name in ['add_text', # raises custom NoEnoughData -- To view, visit https://gerrit.wikimedia.org/r/151291 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I41111456149aab7da02613e581e692d4e1058b31 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

9 years, 8 months

[Gerrit] py2.6 test fix: welcome breaks unreliably - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: py2.6 test fix: welcome breaks unreliably ...................................................................... py2.6 test fix: welcome breaks unreliably Change-Id: I049a4c864a25972fad5e13399f186c649da67331 --- M tests/script_tests.py 1 file changed, 1 insertion(+), 0 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/script_tests.py b/tests/script_tests.py index a6a9398..b371904 100644 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -55,6 +55,7 @@ and check_script_deps(name) and name != 'imageuncat.py' # this halts indefinitely and name != 'watchlist.py' # result depends on speed + and name != 'welcome.py' # result depends on speed and name != 'script_wui.py' # depends on lua compiling ] return script_list -- To view, visit https://gerrit.wikimedia.org/r/151269 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I049a4c864a25972fad5e13399f186c649da67331 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: jenkins-bot <>

9 years, 8 months

[Gerrit] call handleArgs before local args are processed - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: call handleArgs before local args are processed ...................................................................... call handleArgs before local args are processed Change-Id: I0f710087291d74ab423f44088551708f97a21ff8 --- M scripts/watchlist.py 1 file changed, 2 insertions(+), 1 deletion(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/scripts/watchlist.py b/scripts/watchlist.py index 2509088..c3d685c 100755 --- a/scripts/watchlist.py +++ b/scripts/watchlist.py @@ -137,10 +137,11 @@ def main(): + local_args = pywikibot.handleArgs() all = False new = False sysop = False - for arg in pywikibot.handleArgs(): + for arg in local_args: if arg in ('-all', '-update'): all = True elif arg == '-new': -- To view, visit https://gerrit.wikimedia.org/r/151038 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I0f710087291d74ab423f44088551708f97a21ff8 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: jenkins-bot <>

9 years, 8 months

[Gerrit] Script test fixes for py2.6 - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Script test fixes for py2.6 ...................................................................... Script test fixes for py2.6 unittest2 on py2.6 on travis reports the following warning and returns a non-zero exit code for the test suite when a test is unexpectedly successful. DeprecationWarning: Use of a TestResult without an addUnexpectedSuccess method is deprecated Several of our tests fail only on one site (en.wp, ar.wp or wd), or on other conditions which cant be predicted. This test removes use of unexpected success from the tests. Change-Id: Ic2a4be02871f9eda2c40744935392985cd7a6a01 --- M tests/script_tests.py 1 file changed, 7 insertions(+), 6 deletions(-) Approvals: John Vandenberg: Verified; Looks good to me, approved jenkins-bot: Verified diff --git a/tests/script_tests.py b/tests/script_tests.py index 038653e..a6a9398 100644 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -12,6 +12,7 @@ import time import subprocess import pywikibot +from pywikibot import config from tests.utils import unittest, PywikibotTestCase @@ -53,6 +54,8 @@ and not name.startswith('_') # skip __init__.py and _* and check_script_deps(name) and name != 'imageuncat.py' # this halts indefinitely + and name != 'watchlist.py' # result depends on speed + and name != 'script_wui.py' # depends on lua compiling ] return script_list @@ -205,7 +208,6 @@ no_args_expected_results) if script_name in ['add_text', # raises custom NoEnoughData 'checkimages', # bug 68613 - 'cfd', # bug 69015 'commonscat', # raises custom NoEnoughData 'claimit', # bug 68657 - zero output 'data_ingestion', # bug 68611 @@ -214,15 +216,14 @@ 'flickrripper', # bug 68606 (and deps) 'imagerecat', # bug 68658 'imagetransfer', # bug 68659 - 'lonelypages', # raises custom Exception - 'misspelling', # bug 68665 'pagefromfile', # bug 68660 - 'script_wui', # bug 68797 'template', # bug 68661 - zero output 'transferbot', # raises custom Exception 'upload', # raises custom ValueError - 'watchlist', # fails on Wikidata only - ]: + ] or \ + (config.family == 'wikidata' and script_name == 'lonelypages') or \ + ((config.family != 'wikipedia' or config.mylang != 'en') and script_name == 'cfd') or \ + (config.family == 'wikipedia' and config.mylang != 'en' and script_name == 'misspelling'): dct[test_name] = unittest.expectedFailure(dct[test_name]) dct[test_name].__doc__ = \ 'Test running ' + script_name + ' without arguments.' -- To view, visit https://gerrit.wikimedia.org/r/151255 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ic2a4be02871f9eda2c40744935392985cd7a6a01 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: jenkins-bot <>

9 years, 8 months

[Gerrit] Add expected failures for build errors on travis - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Add expected failures for build errors on travis ...................................................................... Add expected failures for build errors on travis Change-Id: Icde3cc4b9bc427f9ba022fa43afb092ffb0f99cc --- M tests/script_tests.py 1 file changed, 6 insertions(+), 0 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/script_tests.py b/tests/script_tests.py index 52ef10e..038653e 100644 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -90,6 +90,7 @@ 'solve_disambiguation', 'unusedfiles', 'upload', + 'watchlist', 'welcome', ] @@ -194,6 +195,7 @@ 'data_ingestion', # bug 68611 'flickrripper', # bug 68606 (and others) 'replicate_wiki', # bug 68664 + 'script_wui', # Failing on travis-ci ]: dct[test_name] = unittest.expectedFailure(dct[test_name]) dct[test_name].__doc__ = 'Test running ' + script_name + '.' @@ -203,6 +205,7 @@ no_args_expected_results) if script_name in ['add_text', # raises custom NoEnoughData 'checkimages', # bug 68613 + 'cfd', # bug 69015 'commonscat', # raises custom NoEnoughData 'claimit', # bug 68657 - zero output 'data_ingestion', # bug 68611 @@ -211,11 +214,14 @@ 'flickrripper', # bug 68606 (and deps) 'imagerecat', # bug 68658 'imagetransfer', # bug 68659 + 'lonelypages', # raises custom Exception + 'misspelling', # bug 68665 'pagefromfile', # bug 68660 'script_wui', # bug 68797 'template', # bug 68661 - zero output 'transferbot', # raises custom Exception 'upload', # raises custom ValueError + 'watchlist', # fails on Wikidata only ]: dct[test_name] = unittest.expectedFailure(dct[test_name]) dct[test_name].__doc__ = \ -- To view, visit https://gerrit.wikimedia.org/r/151122 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Icde3cc4b9bc427f9ba022fa43afb092ffb0f99cc Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

9 years, 8 months

[Gerrit] Remove imageuncat from script tests - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Remove imageuncat from script tests ...................................................................... Remove imageuncat from script tests imageuncat is causing the travis builds to halt. Also replace 'PIL' with module name 'ImageTk' in dependency check. Change-Id: If188c217f2d193da1b6333f04191e320d2e795b5 --- M tests/script_tests.py 1 file changed, 4 insertions(+), 2 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/script_tests.py b/tests/script_tests.py index 523d882..52ef10e 100644 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -24,7 +24,7 @@ 'script_wui.py': ['crontab', 'lua'], # Note: package 'lunatic-python' provides module 'lua' - 'flickrripper.py': ['PIL', 'flickrapi'], + 'flickrripper.py': ['ImageTk', 'flickrapi'], # Note: 'PIL' is not available via pip2.7 on MS Windows, # however it is available with setuptools. } @@ -51,7 +51,9 @@ script_list = [name[0:-3] for name in dir_list # strip .py if name.endswith('.py') and not name.startswith('_') # skip __init__.py and _* - and check_script_deps(name)] + and check_script_deps(name) + and name != 'imageuncat.py' # this halts indefinitely + ] return script_list script_input = { -- To view, visit https://gerrit.wikimedia.org/r/151091 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: If188c217f2d193da1b6333f04191e320d2e795b5 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

9 years, 8 months

[Gerrit] Fixup for recent standardisation of nowcommons - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Fixup for recent standardisation of nowcommons ...................................................................... Fixup for recent standardisation of nowcommons Fixes I1cc6b5734237d143c8100f3c26c20d78c36924c6 Change-Id: I14ea9f7be0bc0df363a2fe2945a068753645598f --- M scripts/nowcommons.py 1 file changed, 7 insertions(+), 7 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/scripts/nowcommons.py b/scripts/nowcommons.py index 260dcfd..3f2c60e 100644 --- a/scripts/nowcommons.py +++ b/scripts/nowcommons.py @@ -261,7 +261,7 @@ break def getPageGenerator(self): - if use_hash: + if self.getOption('use_hash'): gen = self.useHashGenerator() else: nowCommonsTemplates = [pywikibot.Page(self.site, title, @@ -309,7 +309,7 @@ comment = i18n.translate(self.site, nowCommonsMessage, fallback=True) for page in self.getPageGenerator(): - if use_hash: + if self.getOption('use_hash'): # Page -> Has the namespace | commons image -> Not images_list = page # 0 -> local image, 1 -> commons image page = pywikibot.Page(self.site, images_list[0]) @@ -325,18 +325,18 @@ pywikibot.output(u'File is already on Commons.') continue md5 = localImagePage.getFileMd5Sum() - if use_hash: + if self.getOption('use_hash'): filenameOnCommons = images_list[1] else: filenameOnCommons = self.findFilenameOnCommons( localImagePage) - if not filenameOnCommons and not use_hash: + if not filenameOnCommons and not self.getOption('use_hash'): pywikibot.output(u'NowCommons template not found.') continue commonsImagePage = pywikibot.ImagePage(commons, 'Image:%s' % filenameOnCommons) if localImagePage.title(withNamespace=False) == \ - commonsImagePage.title(withNamespace=False) and use_hash: + commonsImagePage.title(withNamespace=False) and self.getOption('use_hash'): pywikibot.output( u'The local and the commons images have the same name') if localImagePage.title(withNamespace=False) != \ @@ -378,7 +378,7 @@ # refresh because we want the updated list usingPages = len(list(pywikibot.ImagePage( self.site, page.title()).usingPages())) - if usingPages > 0 and use_hash: + if usingPages > 0 and self.getOption('use_hash'): # just an enter pywikibot.input( u'There are still %s pages with this \ @@ -397,7 +397,7 @@ if md5 == commonsImagePage.getFileMd5Sum(): pywikibot.output( u'The image is identical to the one on Commons.') - if len(localImagePage.getFileVersionHistory()) > 1 and not use_hash: + if len(localImagePage.getFileVersionHistory()) > 1 and not self.getOption('use_hash'): pywikibot.output( u"This image has a version history. Please \ delete it manually after making sure that the \ -- To view, visit https://gerrit.wikimedia.org/r/151098 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I14ea9f7be0bc0df363a2fe2945a068753645598f Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

9 years, 8 months

[Gerrit] Verify each script compiles and runs - change (pywikibot/core)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Verify each script compiles and runs ...................................................................... Verify each script compiles and runs Adds a set of tests covering compilation and execution of scripts where possible, with limited verification of the output of each script. Invoke tests for all dependencies using: $ PYSETUP_TEST_EXTRAS=yes python setup.py test Change-Id: I836b641c5e315abc0c8eebf130885294d26ffd01 --- M .travis.yml M setup.py A tests/script_tests.py 3 files changed, 280 insertions(+), 4 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/.travis.yml b/.travis.yml index dbd5764..91b1031 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,10 @@ - '2.7' - '2.6' +before_install: + - sudo apt-get update -qq + - sudo apt-get install -y python-imaging-tk liblua5.1-dev + install: - mkdir ~/.pywikibot @@ -34,7 +38,7 @@ - secure: kofInMlisiTBt9o/Ustc/vySlkKfxGzGCX2LwA1D2waym8sDTS0o5gMJ5LsrT/BUKwZbe1vLozPHqZrrkQvsdTml+DpZuotzdILs0m0f3BUoexEC6OON5IDljuxFyETrD1Ug44ih5Mc4lVFOdTcBzg501ZmswGwQrBvg/OyEFfE= matrix: - LANGUAGE=en FAMILY=wikipedia - - LANGUAGE=ar FAMILY=wikipedia + - LANGUAGE=ar FAMILY=wikipedia PYSETUP_TEST_EXTRAS=1 - LANGUAGE=test FAMILY=wikidata notifications: diff --git a/setup.py b/setup.py index 069dc46..6682b3f 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ import sys import os +import itertools from ez_setup import use_setuptools use_setuptools() @@ -22,11 +23,24 @@ test_deps = [] testcollector = "tests" +dependencies = ['httplib2>=0.6.0'] + +extra_deps = {} + +script_deps = { + 'script_wui.py': ['irc', 'lunatic-python', 'crontab'], + # Note: None of the 'lunatic-python' repos on github support MS Windows. + 'flickrripper.py': ['PIL', 'flickrapi'], + # Note: 'PIL' is not available via pip2.7 on MS Windows, + # however it is available with setuptools. +} + if sys.version_info[0] == 2: if sys.version_info < (2, 6, 5): raise RuntimeError("ERROR: Pywikibot only runs under Python 2.6.5 or higher") elif sys.version_info[1] == 6: - test_deps = ['unittest2'] + test_deps.append('unittest2') + script_deps['replicate_wiki.py'] = ['argparse'] testcollector = "tests.utils.collector" if sys.version_info[0] == 3: @@ -38,7 +52,6 @@ print("ERROR: Python 3.3 or higher is required!") sys.exit(1) -dependencies = ['httplib2>=0.6.0'] if os.name != 'nt': # See bug 66010, Windows users will have issues # when trying to build the C modules. @@ -60,6 +73,13 @@ python = python.replace("pythonw.exe", "python.exe") # for Windows subprocess.call([python, "generate_user_files.py"]) + +extra_deps.update(script_deps) +# Add script dependencies as test dependencies, +# so scripts can be compiled in test suite. +if 'PYSETUP_TEST_EXTRAS' in os.environ: + test_deps += list(itertools.chain(*(script_deps.values()))) + setup( name='pywikibot', version='2.0b1', @@ -73,8 +93,11 @@ for package in find_packages() if package.startswith('pywikibot.')], install_requires=dependencies, + extras_require=extra_deps, dependency_links=[ - 'https://git.wikimedia.org/zip/?r=pywikibot/externals/httplib2.git&format=gz…' + 'https://git.wikimedia.org/zip/?r=pywikibot/externals/httplib2.git&format=gz…', + 'git+https://github.com/AlereDevices/lunatic-python.git#egg=lunatic-python', + 'git+https://github.com/jayvdb/parse-crontab.git#egg=crontab', ], url='https://www.mediawiki.org/wiki/Pywikibot', download_url='https://github.com/wikimedia/pywikibot-core/archive/master.zip#egg=pywikibo…', diff --git a/tests/script_tests.py b/tests/script_tests.py new file mode 100644 index 0000000..523d882 --- /dev/null +++ b/tests/script_tests.py @@ -0,0 +1,249 @@ +# -*- coding: utf-8 -*- +"""Test that each script can be compiled and executed.""" +# +# (C) Pywikibot team, 2014 +# +# Distributed under the terms of the MIT license. +# +__version__ = '$Id$' + +import os +import sys +import time +import subprocess +import pywikibot +from tests.utils import unittest, PywikibotTestCase + + +base_path = os.path.split(os.path.split(__file__)[0])[0] +pwb_path = os.path.join(base_path, 'pwb.py') +scripts_path = os.path.join(base_path, 'scripts') + + +script_deps = { + 'script_wui.py': ['crontab', 'lua'], + # Note: package 'lunatic-python' provides module 'lua' + + 'flickrripper.py': ['PIL', 'flickrapi'], + # Note: 'PIL' is not available via pip2.7 on MS Windows, + # however it is available with setuptools. +} +if sys.version_info < (2, 7): + script_deps['replicate_wiki.py'] = ['argparse'] + + +def check_script_deps(script_name): + """Detect whether all dependencies are installed.""" + if script_name in script_deps: + for package_name in script_deps[script_name]: + try: + __import__(package_name) + except ImportError as e: + print('%s depends on %s, which isnt available:\n%s' + % (script_name, package_name, e)) + return False + return True + + +def runnable_script_list(scripts_path): + """List of scripts which may be executed.""" + dir_list = os.listdir(scripts_path) + script_list = [name[0:-3] for name in dir_list # strip .py + if name.endswith('.py') + and not name.startswith('_') # skip __init__.py and _* + and check_script_deps(name)] + return script_list + +script_input = { + 'catall': 'q\n', # q for quit + 'disambredir': '\n', # prompts for user to choose action to take + 'editarticle': 'Test page\n', + 'imagetransfer': 'Test page\n', + 'interwiki': 'Test page\n', + 'makecat': 'Test page\n\n', + # 'misspelling': 'q\n', # pressing 'q' doesnt work. bug 68663 + 'replace': 'foo\nbar\n\n\n', # match, replacement, + # Enter to begin, Enter for default summary. + 'shell': '\n', # exits on end of stdin + 'solve_disambiguation': 'Test page\nq\n', + 'upload': 'https://upload.wikimedia.org/wikipedia/commons/8/80/Wikipedia-logo-v2.svg\n…', +} + +auto_run_script_list = [ + 'blockpageschecker', + 'blockreview', + 'casechecker', + 'catall', + 'category_redirect', + 'cfd', + 'clean_sandbox', + 'lonelypages', + 'makecat', + 'misspelling', + 'revertbot', + 'noreferences', + 'nowcommons', + 'script_wui', + 'shell', + 'solve_disambiguation', + 'unusedfiles', + 'upload', + 'welcome', +] + +# Expected result for no arguments +# Some of these are not pretty, but at least they are informative +# and not backtraces starting deep in the pywikibot package. +no_args_expected_results = { + 'add_text': 'NoEnoughData', + 'archivebot': 'NOTE: you must specify a template to run the bot', + 'commonscat': 'add_text.NoEnoughData', + 'create_categories': 'No pages to work on', + 'editarticle': 'Nothing changed', # This masks related bug 68645 but that + # bug is more broadly about config + # rather than editarticle. + 'freebasemappingupload': 'Cannot find ', + 'harvest_template': 'ERROR: Please specify', + 'illustrate_wikidata': 'I need a generator with pages to work on', + 'imageuncat': 'You have to specify the generator ', + 'interwiki': 'does not exist. Skipping.', # 'Test page' does not exist + 'login': 'Logged in on ', + 'replace': 'Press Enter to use this default message', + 'replicate_wiki': 'error: too few arguments', + 'script_wui': 'Pre-loading all relevant page contents', + 'shell': 'Welcome to the', + 'spamremove': 'No spam site specified', + 'transferbot': 'Target site not different from source site', # Bug 68662 + 'version': 'unicode test: ', + 'watchlist': 'Retrieving watchlist', + + # The following auto-run and typically cant be validated, + # however these strings are very likely to exist within + # the timeout of 5 seconds. + 'makecat': '(Default is [[', + 'revertbot': 'Fetching new batch of contributions', + 'upload': 'ERROR: Upload error', +} + + +def execute(command, data_in=None, timeout=0): + """Execute a command and capture outputs.""" + options = { + 'stdout': subprocess.PIPE, + 'stderr': subprocess.PIPE + } + if data_in is not None: + options['stdin'] = subprocess.PIPE + + p = subprocess.Popen(command, **options) + if data_in is not None: + p.stdin.write(data_in) + waited = 0 + while waited < timeout and p.poll() is None: + time.sleep(1) + waited += 1 + if timeout and p.poll() is None: + p.kill() + data_out = p.communicate() + return {'exit_code': p.returncode, + 'stdout': data_out[0], + 'stderr': data_out[1]} + + +class TestScriptMeta(type): + + """Test meta class.""" + + def __new__(cls, name, bases, dct): + """Create the new class.""" + def test_execution(script_name, args=None, expected_results=None): + def testScript(self): + cmd = [sys.executable, pwb_path, script_name] + + if args: + cmd += args + + data_in = script_input.get(script_name) + + timeout = 0 + if script_name in auto_run_script_list: + timeout = 5 + result = execute(cmd, data_in, timeout=timeout) + + if expected_results and script_name in expected_results: + if expected_results[script_name] is not None: + self.assertIn(expected_results[script_name], + result['stderr']) + elif (args and '-help' in args) or \ + script_name not in auto_run_script_list: + self.assertEqual(result['stderr'], '') + self.assertIn('Global arguments available for all', + result['stdout']) + self.assertEqual(result['exit_code'], 0) + self.assertNotIn('Traceback (most recent call last)', + result['stderr']) + return testScript + + for script_name in runnable_script_list(scripts_path): + test_name = 'test_' + script_name + '_execution' + dct[test_name] = test_execution(script_name, ['-help']) + if script_name in ['shell', 'version', + 'checkimages', # bug 68613 + 'data_ingestion', # bug 68611 + 'flickrripper', # bug 68606 (and others) + 'replicate_wiki', # bug 68664 + ]: + dct[test_name] = unittest.expectedFailure(dct[test_name]) + dct[test_name].__doc__ = 'Test running ' + script_name + '.' + + test_name = 'test_' + script_name + '_no_args' + dct[test_name] = test_execution(script_name, ['-simulate'], + no_args_expected_results) + if script_name in ['add_text', # raises custom NoEnoughData + 'checkimages', # bug 68613 + 'commonscat', # raises custom NoEnoughData + 'claimit', # bug 68657 - zero output + 'data_ingestion', # bug 68611 + 'disambredir', # quittable auto-run with + # highly variable output. + 'flickrripper', # bug 68606 (and deps) + 'imagerecat', # bug 68658 + 'imagetransfer', # bug 68659 + 'pagefromfile', # bug 68660 + 'script_wui', # bug 68797 + 'template', # bug 68661 - zero output + 'transferbot', # raises custom Exception + 'upload', # raises custom ValueError + ]: + dct[test_name] = unittest.expectedFailure(dct[test_name]) + dct[test_name].__doc__ = \ + 'Test running ' + script_name + ' without arguments.' + + return type.__new__(cls, name, bases, dct) + + +class TestScript(PywikibotTestCase): + + """Test cases for scripts.""" + + __metaclass__ = TestScriptMeta + + def setUp(self): + """Prepare the environment for running the pwb.py script.""" + self.old_pywikibot_dir = None + if 'PYWIKIBOT2_DIR' in os.environ: + self.old_pywikibot_dir = os.environ['PYWIKIBOT2_DIR'] + os.environ['PYWIKIBOT2_DIR'] = pywikibot.config.base_dir + + def tearDown(self): + """Restore the environment after running the pwb.py script.""" + del os.environ['PYWIKIBOT2_DIR'] + if self.old_pywikibot_dir: + os.environ['PYWIKIBOT2_DIR'] = self.old_pywikibot_dir + + +if __name__ == '__main__': + try: + unittest.main() + except SystemExit: + pass -- To view, visit https://gerrit.wikimedia.org/r/149545 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I836b641c5e315abc0c8eebf130885294d26ffd01 Gerrit-PatchSet: 8 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Ricordisamoa <ricordisamoa(a)openmailbox.org> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

9 years, 8 months

[Gerrit] Update Wikia's MediaWiki version to 1.19.18 - change (pywikibot/compat)

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. Change subject: Update Wikia's MediaWiki version to 1.19.18 ...................................................................... Update Wikia's MediaWiki version to 1.19.18 Change-Id: Ie4b8f70f0077530835427e368d7d92144c1dd02c --- M families/lyricwiki_family.py M families/wikia_family.py M families/wowwiki_family.py 3 files changed, 4 insertions(+), 4 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/families/lyricwiki_family.py b/families/lyricwiki_family.py index 2bd0688..95fb8c0 100644 --- a/families/lyricwiki_family.py +++ b/families/lyricwiki_family.py @@ -34,7 +34,7 @@ self.namespaces[503] = { '_default': u'Blog talk', } def version(self, code): - return '1.19.17' + return '1.19.18' def scriptpath(self, code): return '' diff --git a/families/wikia_family.py b/families/wikia_family.py index 2a33f15..15a9384 100644 --- a/families/wikia_family.py +++ b/families/wikia_family.py @@ -39,7 +39,7 @@ return u'www.wikia.com' def version(self, code): - return "1.19.17" + return "1.19.18" def scriptpath(self, code): return '' diff --git a/families/wowwiki_family.py b/families/wowwiki_family.py index 5acce57..509b528 100644 --- a/families/wowwiki_family.py +++ b/families/wowwiki_family.py @@ -330,7 +330,7 @@ self.namespaces[801] = { 'en': u'Poll talk' } - + self.namespaces[902] = { 'es': u'Layout' } @@ -352,4 +352,4 @@ return '' def version(self, code): - return '1.19.17' + return '1.19.18' -- To view, visit https://gerrit.wikimedia.org/r/150792 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ie4b8f70f0077530835427e368d7d92144c1dd02c Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/compat Gerrit-Branch: master Gerrit-Owner: Grunny <mwgrunny(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

9 years, 8 months

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

Pywikibot-commits August 2014