Pywikibot-commits December 2016

pywikibot-commits@lists.wikimedia.org

2 participants
59 discussions

[Gerrit] pywikibot/core[master]: PetScan page generator

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/327753 ) Change subject: PetScan page generator ...................................................................... PetScan page generator Bug: T60814 Change-Id: I23044630a34c7cc4645524b0291448a338e2f176 --- M pywikibot/pagegenerators.py M tests/pagegenerators_tests.py 2 files changed, 120 insertions(+), 0 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index eafd14b..3db16a7 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -27,6 +27,7 @@ import codecs import datetime import itertools +import json import re import sys import time @@ -46,12 +47,21 @@ intersect_generators, IteratorNextMixin, filter_unique, + PY2, ) from pywikibot import date, config, i18n, xmlreader +from pywikibot.comms import http from pywikibot.exceptions import ArgumentDeprecationWarning, UnknownExtension from pywikibot.logentries import LogEntryFactory from pywikibot.proofreadpage import ProofreadPage + +if PY2: + from urllib import urlencode + import urlparse +else: + import urllib.parse as urlparse + from urllib.parse import urlencode if sys.version_info[0] > 2: basestring = (str, ) @@ -2764,6 +2774,92 @@ yield pywikibot.ItemPage(repo, item['id']) +class PetScanPageGenerator(object): + """Queries PetScan (https://petscan.wmflabs.org/) to generate pages.""" + + def __init__(self, categories, subset_combination=True, namespaces=None, + site=None, extra_options=None): + """ + Constructor. + + :param categories: List of categories to retrieve pages from + (as strings) + :param subset_combination: Combination mode. + If True, returns the intersection of the results of the categories, + else returns the union of the results of the categories + :param namespaces: List of namespaces to search in + (default is None, meaning all namespaces) + :param site: Site to operate on + (default is the default site from the user config) + :param extra_options: Dictionary of extra options to use (optional) + """ + if site is None: + site = pywikibot.Site() + + self.site = site + self.opts = self.buildQuery(categories, subset_combination, + namespaces, extra_options) + + def buildQuery(self, categories, subset_combination, namespaces, + extra_options): + """ + Get the querystring options to query PetScan. + + :param categories: List of categories (as strings) + :param subset_combination: Combination mode. + If True, returns the intersection of the results of the categories, + else returns the union of the results of the categories + :param namespaces: List of namespaces to search in + :param extra_options: Dictionary of extra options to use + :return: Dictionary of querystring parameters to use in the query + """ + extra_options = extra_options or {} + + query = { + 'language': self.site.lang, + 'project': self.site.family, + 'combination': 'subset' if subset_combination else 'union', + 'categories': '\r\n'.join(categories), + 'format': 'json', + 'doit': '' + } + + # test wikipedia + if self.site.code == 'test' and self.site.family == 'test': + query['language'] = 'test' + query['project'] = 'wikipedia' + + if namespaces: + for namespace in namespaces: + query['ns[{0}]'.format(int(namespace))] = 1 + + query_final = query.copy() + query_final.update(extra_options) + + return query_final + + def query(self): + """Query PetScan.""" + url = urlparse.urlunparse(('https', # scheme + 'petscan.wmflabs.org', # netloc + '', # path + '', # params + urlencode(self.opts), # query + '')) # fragment + + req = http.fetch(url) + j = json.loads(req.content) + raw_pages = j['*'][0]['a']['*'] + for raw_page in raw_pages: + yield raw_page + + def __iter__(self): + for raw_page in self.query(): + page = pywikibot.Page(self.site, raw_page['title'], + int(raw_page['namespace'])) + yield page + + # Deprecated old names available for compatibility with compat. ImageGenerator = redirect_func(FileGenerator, old_name='ImageGenerator') UnCategorizedTemplatesGenerator = redirect_func( diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py index f7b7b31..7abf648 100755 --- a/tests/pagegenerators_tests.py +++ b/tests/pagegenerators_tests.py @@ -355,6 +355,30 @@ self.assertPagelistTitles(gen, titles=expect_3, site=site) +class PetScanPageGeneratorTestCase(TestCase): + + """Test PetScanPageGenerator.""" + + family = 'test' + code = 'test' + + def test_petscan(self): + """Test PetScanPageGenerator.""" + site = self.get_site() + gen = pagegenerators.PetScanPageGenerator(['Pywikibot Protect Test'], True, None, site) + self.assertPagelistTitles(gen, titles=('User:Sn1per/ProtectTest1', + 'User:Sn1per/ProtectTest2'), site=site) + + gen = pagegenerators.PetScanPageGenerator(['Pywikibot Protect Test'], False, None, site) + self.assertPagelistTitles(gen, titles=('User:Sn1per/ProtectTest1', + 'User:Sn1per/ProtectTest2'), site=site) + + gen = pagegenerators.PetScanPageGenerator(['Pywikibot PetScan Test', + 'Pywikibot Category That Needs&ToBe!Encoded', + 'Test'], True, None, site) + self.assertPagelistTitles(gen, titles=('User:Sn1per/PetScanTest1',), site=site) + + class TestRepeatingGenerator(RecentChangesTestCase): """Test RepeatingGenerator.""" -- To view, visit https://gerrit.wikimedia.org/r/327753 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I23044630a34c7cc4645524b0291448a338e2f176 Gerrit-PatchSet: 5 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Sn1per <geofbot(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com> Gerrit-Reviewer: Sn1per <geofbot(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

7 years, 4 months

[Gerrit] pywikibot/core[master]: Add fi:wikivoyage and sort by current article count

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/327766 ) Change subject: Add fi:wikivoyage and sort by current article count ...................................................................... Add fi:wikivoyage and sort by current article count Fix also pycodestyle (former PEP8) E402 problem Bug: T153470 Change-Id: Id9bc980c7a9cfb21063597a3d5eae11c31d8040c --- M pywikibot/families/wikivoyage_family.py 1 file changed, 5 insertions(+), 6 deletions(-) Approvals: Lokal Profil: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/families/wikivoyage_family.py b/pywikibot/families/wikivoyage_family.py index 1ad8583..eb0947e 100644 --- a/pywikibot/families/wikivoyage_family.py +++ b/pywikibot/families/wikivoyage_family.py @@ -5,13 +5,12 @@ # # Distributed under the terms of the MIT license. # +# The new wikivoyage family that is hosted at wikimedia from __future__ import absolute_import, unicode_literals -__version__ = '$Id$' - -# The new wikivoyage family that is hosted at wikimedia - from pywikibot import family + +__version__ = '$Id$' class Family(family.SubdomainFamily, family.WikimediaFamily): @@ -23,8 +22,8 @@ def __init__(self): """Constructor.""" self.languages_by_size = [ - 'en', 'de', 'fa', 'it', 'fr', 'ru', 'pl', 'nl', 'pt', 'es', 'he', - 'zh', 'vi', 'sv', 'el', 'ro', 'uk', + 'en', 'de', 'fa', 'it', 'fr', 'ru', 'pl', 'nl', 'pt', 'fi', 'es', + 'zh', 'he', 'vi', 'sv', 'el', 'ro', 'uk', ] super(Family, self).__init__() -- To view, visit https://gerrit.wikimedia.org/r/327766 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Id9bc980c7a9cfb21063597a3d5eae11c31d8040c Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Magul <tomasz.magulski(a)gmail.com> Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Lokal Profil <lokal.profil(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

7 years, 4 months

[Gerrit] pywikibot/core[master]: Use user-based configuration settings in script_wui.py

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/206790 ) Change subject: Use user-based configuration settings in script_wui.py ...................................................................... Use user-based configuration settings in script_wui.py And also a nice error message when it doesn't exist. Bug: T70797 Change-Id: I7486e66df0c05b490f1e8cc95c28b74e370fe90a --- M scripts/script_wui.py 1 file changed, 19 insertions(+), 12 deletions(-) Approvals: Ladsgroup: Looks good to me, approved jenkins-bot: Verified diff --git a/scripts/script_wui.py b/scripts/script_wui.py index 616e35f..ba75271 100755 --- a/scripts/script_wui.py +++ b/scripts/script_wui.py @@ -73,7 +73,6 @@ import datetime import gc import logging -import os import re import resource import sys @@ -107,14 +106,11 @@ bot_config = { - 'BotName': pywikibot.config.usernames[pywikibot.config.family][pywikibot.config.mylang], + 'BotName': "{username}", - # protected !!! ('CSS' or other semi-protected page is essential here) - 'ConfCSSshell': 'User:DrTrigon/DrTrigonBot/script_wui-shell.css', - 'ConfCSScrontab': u'User:DrTrigon/DrTrigonBot/script_wui-crontab.css', - - # (may be protected but not that important... 'CSS' is not needed here !!!) - 'ConfCSSoutput': u'User:DrTrigonBot/Simulation', + 'ConfCSSshell': u'User:{username}/script_wui-shell.css', + 'ConfCSScrontab': u'User:{username}/script_wui-crontab.css', + 'ConfCSSoutput': u'User:{username}/Simulation', 'CRONMaxDelay': 5 * 60.0, # check all ~5 minutes @@ -157,15 +153,20 @@ } pywikibot.output(u'** Pre-loading all relevant page contents') for item in self.refs: - # security; first check if page is protected, reject any data if not - if os.path.splitext(self.refs[item].title().lower())[1] not in ['.css', '.js']: + # First check if page is protected, reject any data if not + parts = self.refs[item].title().lower().rsplit('.') + if len(parts) == 1 or parts[1] not in ['.css', '.js']: raise ValueError(u'%s config %s = %s is not a secure page; ' u'it should be a css or js userpage which are ' u'automatically semi-protected.' % (self.__class__.__name__, item, self.refs[item])) - self.refs[item].get(force=True) # load all page contents - + try: + self.refs[item].get(force=True) # load all page contents + except pywikibot.NoPage: + pywikibot.error("The configuation page %s doesn't exists" + % self.refs[item].title(asLink=True)) + raise # init background timer pywikibot.output(u'** Starting crontab background timer thread') self.on_timer() @@ -326,6 +327,12 @@ site = pywikibot.Site() site.login() chan = '#' + site.code + '.' + site.family.name + + bot_user_name = pywikibot.config.usernames[pywikibot.config.family][pywikibot.config.mylang] + for key, value in bot_config.items(): + if hasattr(value, 'format'): + bot_config[key] = value.format(username=bot_user_name) + bot = ScriptWUIBot(site, chan, site.user() + "_WUI", "irc.wikimedia.org") try: bot.start() -- To view, visit https://gerrit.wikimedia.org/r/206790 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I7486e66df0c05b490f1e8cc95c28b74e370fe90a Gerrit-PatchSet: 8 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Ladsgroup <Ladsgroup(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <Ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Pppery <mapreader(a)olum.org> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

7 years, 4 months

[Gerrit] pywikibot/core[master]: Fix regex for error message in siteinfo request

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/327551 ) Change subject: Fix regex for error message in siteinfo request ...................................................................... Fix regex for error message in siteinfo request Allow new format of error message introduced in https://phabricator.wikimedia.org/rMW4e6810e4a Bug: T153325 Change-Id: I1e2e7dbd711f385fd6ee891f08fa016ddc8fb38c --- M pywikibot/site.py 1 file changed, 2 insertions(+), 2 deletions(-) Approvals: Dalba: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/site.py b/pywikibot/site.py index 92245fd..12a40f4 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -1402,8 +1402,8 @@ All values of the siteinfo property 'general' are directly available. """ - WARNING_REGEX = re.compile(u"^Unrecognized values? for parameter " - u"'siprop': ([^,]+(?:, [^,]+)*)$") + WARNING_REGEX = re.compile('^Unrecognized values? for parameter ' + '["\']siprop["\']: (.+?)\.?$') # Until we get formatversion=2, we have to convert empty-string properties # into booleans so they are easier to use. -- To view, visit https://gerrit.wikimedia.org/r/327551 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I1e2e7dbd711f385fd6ee891f08fa016ddc8fb38c Gerrit-PatchSet: 4 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Magul <tomasz.magulski(a)gmail.com> Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Lokal Profil <lokal.profil(a)gmail.com> Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

7 years, 4 months

[Gerrit] pywikibot/core[master]: Add new error code for can't execute special page

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/327559 ) Change subject: Add new error code for can't execute special page ...................................................................... Add new error code for can't execute special page Bug: T153323 Change-Id: Ic6fed1177ced52040bfe214170b337cee286f97b --- M tests/site_tests.py 1 file changed, 2 insertions(+), 1 deletion(-) Approvals: Dalba: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/site_tests.py b/tests/site_tests.py index a504d44..5de88b5 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -811,7 +811,8 @@ try: unwatchedpages = list(mysite.unwatchedpages(total=10)) except api.APIError as error: - if error.code == 'gqpspecialpage-cantexecute': + if error.code in ('specialpage-cantexecute', + 'gqpspecialpage-cantexecute'): # User must have correct permissions to use Special:UnwatchedPages raise unittest.SkipTest(error) raise -- To view, visit https://gerrit.wikimedia.org/r/327559 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ic6fed1177ced52040bfe214170b337cee286f97b Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Magul <tomasz.magulski(a)gmail.com> Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Lokal Profil <lokal.profil(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

7 years, 4 months

[Gerrit] pywikibot/core[master]: Raise AssertionError on any failure in assertSite

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/327514 ) Change subject: Raise AssertionError on any failure in assertSite ...................................................................... Raise AssertionError on any failure in assertSite Bug: T151368 Change-Id: I673f284de5da2a83f9b8c72475d24b5625a3b274 --- M tests/site_detect_tests.py 1 file changed, 5 insertions(+), 1 deletion(-) Approvals: Lokal Profil: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/site_detect_tests.py b/tests/site_detect_tests.py index f8bf337..c808b0a 100644 --- a/tests/site_detect_tests.py +++ b/tests/site_detect_tests.py @@ -61,7 +61,11 @@ @type url: str @raises AssertionError: Site under url is not MediaWiki powered """ - self.assertIsInstance(MWSite(url), MWSite) + try: + self.assertIsInstance(MWSite(url), MWSite) + except (AttributeError, ConnectionError, RuntimeError, ServerError, + Timeout): + raise AssertionError def assertNoSite(self, url): """ -- To view, visit https://gerrit.wikimedia.org/r/327514 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I673f284de5da2a83f9b8c72475d24b5625a3b274 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Magul <tomasz.magulski(a)gmail.com> Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Lokal Profil <lokal.profil(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

7 years, 4 months

[Gerrit] pywikibot/core[master]: Fix www.wikichristian.org and remove www.lojban.org detectio...

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/326358 ) Change subject: Fix www.wikichristian.org and remove www.lojban.org detection tests ...................................................................... Fix www.wikichristian.org and remove www.lojban.org detection tests Also radical simplification of whole site_detect_tests module. Bug: T151368 Change-Id: I170d0f136e8f0983aa82eb51bea6d28fbec50343 --- M tests/__init__.py M tests/site_detect_tests.py 2 files changed, 242 insertions(+), 211 deletions(-) Approvals: Lokal Profil: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/__init__.py b/tests/__init__.py index 7996ec4..1d51dcb 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -158,9 +158,6 @@ 'textlib': [ 'test_interwiki_format', # example; very slow test ], - 'site_detect': [ - 'test_IWM', # very slow and tests include unnecessary sites - ], 'weblib': [ 'testWebCiteOlder', # fails. T110640 ], diff --git a/tests/site_detect_tests.py b/tests/site_detect_tests.py index 385a95d..f8bf337 100644 --- a/tests/site_detect_tests.py +++ b/tests/site_detect_tests.py @@ -5,266 +5,300 @@ # # Distributed under the terms of the MIT license. # + from __future__ import absolute_import, unicode_literals -__version__ = '$Id$' - -from requests.exceptions import Timeout +from requests.exceptions import ConnectionError, Timeout from pywikibot.exceptions import ServerError from pywikibot.site_detect import MWSite -from pywikibot.tools import MediaWikiVersion from tests.aspects import unittest, TestCase -class TestWikiSiteDetection(TestCase): +__version__ = '$Id$' - """Test Case for MediaWiki detection and site object creation.""" - # This list is intentionally shared between all classes - _urls_tested = set() - # Whether it allows multiple tests of the same URL - allow_multiple = False +def _retry_few_times(retry_limit): + """ + Decorator to retry test on failure. - def setUp(self): - """Set up test.""" - self.skips = {} - self.failures = {} - self.errors = {} - self.passes = {} - self.all = [] - # reset after end of test - self._previous_multiple = self.allow_multiple - super(TestWikiSiteDetection, self).setUp() + Swallow AssertionError retry_limit times before failing test. - def tearDown(self): - """Tear Down test.""" - def norm(url): - res = None - typ = -1 - for pos, result in enumerate([self.passes, self.errors, - self.failures, self.skips]): - if url in result: - assert res is None - res = result[url] - typ = pos - if res is None: - typ = len(PREFIXES) - 1 - res = 'Missing' - assert 0 <= pos < len(PREFIXES) - return typ, url, res + @param retry_limit: Retry limit before failing test + @type retry_limit: int + @return: a decorator to retry test on failure + @rtype: function + @raises AssertionError: all retries of test failed + """ + def actual_decorator(wrapped_func): + def wrapper_func(*args, **kwargs): + for retry_no in range(1, retry_limit + 1): + try: + wrapped_func(*args, **kwargs) + except AssertionError: + if retry_no == retry_limit: + raise + except: + raise + else: + return + return wrapper_func + return actual_decorator - self.allow_multiple = self._previous_multiple - super(TestWikiSiteDetection, self).tearDown() - print('Out of %d sites, %d tests passed, %d tests failed, ' - '%d tests skiped and %d tests raised an error' - % (len(self.all), len(self.passes), len(self.failures), - len(self.skips), len(self.errors)) - ) - PREFIXES = ['PASS', 'ERR ', 'FAIL', 'SKIP', 'MISS'] +class SiteDetectionTestCase(TestCase): - sorted_all = sorted((norm(url) for url in self.all), - key=lambda item: item[0]) - width = max(len(item[1]) for item in sorted_all) - print('Results:\n' + '\n'.join( - '{0} {1:{3}} : {2}'.format(PREFIXES[i[0]], i[1], i[2], width) - for i in sorted_all)) + """Testcase for MediaWiki detection and site object creation.""" - def _wiki_detection(self, url, result): - """Perform one load test.""" - self.all += [url] - if url in self._urls_tested: - msg = 'Testing URL "{0}" multiple times!'.format(url) - if self.allow_multiple: - print(msg) - else: - self.errors[url] = msg - return - self._urls_tested.add(url) - try: - site = MWSite(url) - except (ServerError, Timeout) as e: - self.skips[url] = e - return - except Exception as e: - print('failure {0} on {1}: {2}'.format( - url, type(e), e)) - self.errors[url] = e - return - try: - if result is None: - self.assertIsNone(site) - else: - self.assertIsInstance(site, result) - self.passes[url] = site - except AssertionError as error: - self.failures[url] = error + net = True def assertSite(self, url): - """Assert a MediaWiki site can be loaded from the url.""" - self._wiki_detection(url, MWSite) + """ + Assert a MediaWiki site can be loaded from the url. + + @param url: Url of tested site + @type url: str + @raises AssertionError: Site under url is not MediaWiki powered + """ + self.assertIsInstance(MWSite(url), MWSite) def assertNoSite(self, url): - """Assert a url is not a MediaWiki site.""" - self._wiki_detection(url, None) + """ + Assert a url is not a MediaWiki site. - def assertAllPass(self): - """Assert that all urls were detected as a MediaWiki site.""" - self.assertEqual(set(self.passes), set(self.all) - set(self.skips)) - self.assertEqual(self.failures, {}) - self.assertEqual(self.errors, {}) - - def assertAllError(self): - """Assert that all urls were not detected as a MediaWiki site.""" - self.assertEqual(self.passes, {}) - self.assertEqual(self.failures, {}) - self.assertEqual(set(self.errors), set(self.all) - set(self.skips)) + @param url: Url of tested site + @type url: str + @raises AssertionError: Site under url is MediaWiki powered + """ + self.assertRaises((AttributeError, ConnectionError, RuntimeError, + ServerError, Timeout), MWSite, url) -class InterWikiMapDetection(TestWikiSiteDetection): +class StandardVersionSiteTestCase(SiteDetectionTestCase): - """Test all urls on the interwiki map.""" + """Test detection of MediaWiki sites.""" - family = 'meta' - code = 'meta' - net = True - - allow_multiple = True - - def test_IWM(self): - """Test the load_site method for MW sites on the IWM list.""" - data = self.get_site().siteinfo['interwikimap'] - for item in data: - if 'local' not in item: - url = item['url'] - self.all += [url] - try: - site = MWSite(url) - except Exception as error: - print('failed to load ' + url) - self.errors[url] = error - continue - if type(site) is MWSite: - try: - version = site.version - except Exception as error: - print('failed to get version of ' + url) - self.errors[url] = error - else: - try: - self.assertIsInstance(version, MediaWikiVersion) - self.passes[url] = site - except AssertionError as error: - print('failed to parse version of ' + url) - self.failures[url] = error - - -class SiteDetectionTestCase(TestWikiSiteDetection): - - """Test all urls on the interwiki map.""" - - net = True - - def test_detect_site(self): - """Test detection of MediaWiki sites.""" + def test_hrwiki(self): + """Test detection of MediaWiki sites for www.hrwiki.org.""" self.assertSite('http://www.hrwiki.org/index.php/$1') # v 1.15 + + def test_proofwiki(self): + """Test detection of MediaWiki sites for www.proofwiki.org.""" self.assertSite('http://www.proofwiki.org/wiki/$1') + + def test_ck_wissen(self): + """Test detection of MediaWiki sites for www.ck-wissen.de.""" self.assertSite( 'http://www.ck-wissen.de/ckwiki/index.php?title=$1') + + def test_citizendium(self): + """Test detection of MediaWiki sites for en.citizendium.org.""" self.assertSite('http://en.citizendium.org/wiki/$1') - self.assertSite( - 'http://www.lojban.org/tiki/tiki-index.php?page=$1') + + @_retry_few_times(10) + def test_wikichristian(self): + """Test detection of MediaWiki sites for www.wikichristian.org. + + Server that hosts www.wikichristian.org is unreliable - it occasionally + responding with 500 error (see: T151368). + """ self.assertSite('http://www.wikichristian.org/index.php?title=$1') + + def test_wikifur(self): + """Test detection of MediaWiki sites for en.wikifur.com.""" self.assertSite('https://en.wikifur.com/wiki/$1') + + def test_bluwiki(self): + """Test detection of MediaWiki sites for bluwiki.com.""" self.assertSite('http://bluwiki.com/go/$1') + + def test_kino_skirpov(self): + """Test detection of MediaWiki sites for kino.skripov.com.""" self.assertSite('http://kino.skripov.com/index.php/$1') - self.assertAllPass() - def test_wikisophia(self): - """Test wikisophia.org which has redirect problems.""" - # /index.php?title=$1 reports 404, however a wiki exists there, - # but the API is also hidden. - self.assertNoSite('http://wikisophia.org/index.php?title=$1') - self.assertAllError() - def test_pre_114_sites(self): - """Test pre 1.14 sites which should be detected as unsupported.""" - # v1.12 - self.assertNoSite('http://www.livepedia.gr/index.php?title=$1') - # v1.11 - self.assertNoSite('http://www.wikifon.org/$1') - self.assertNoSite('http://glossary.reuters.com/index.php?title=$1') +class NonStandardVersionSiteTestCase(SiteDetectionTestCase): + + """Test non-standard version string sites.""" + + def test_gentoo(self): + """Test detection of MediaWiki sites for wiki.gentoo.org.""" + self.assertSite('https://wiki.gentoo.org/wiki/$1') + + def test_arabeyes(self): + """Test detection of MediaWiki sites for wiki.arabeyes.org.""" + self.assertSite('http://wiki.arabeyes.org/$1') + + def test_tfwiki(self): + """Test detection of MediaWiki sites for tfwiki.net.""" + self.assertSite('http://tfwiki.net/wiki/$1') + + +class Pre114SiteTestCase(SiteDetectionTestCase): + + """Test pre 1.14 sites which should be detected as unsupported.""" + + def test_livepedia(self): + """Test detection of MediaWiki sites for www.livepedia.gr.""" + self.assertNoSite( + 'http://www.livepedia.gr/index.php?title=$1') # v1.12 + + def test_wikifon(self): + """Test detection of MediaWiki sites for www.wikifon.org.""" + self.assertNoSite('http://www.wikifon.org/$1') # v1.11 + + def test_reuters(self): + """Test detection of MediaWiki sites for glossary.reuters.com.""" + self.assertNoSite( + 'http://glossary.reuters.com/index.php?title=$1') # v1.11 + + def test_wikitree(self): + """Test detection of MediaWiki sites for wikitree.org.""" # v1.11, with no query module self.assertNoSite('http://wikitree.org/index.php?title=$1') - # v1.9 - self.assertNoSite('http://www.wikinvest.com/$1') - self.assertAllError() - def test_non_standard_version_sites(self): - """Test non-standard version string sites.""" - self.assertSite('https://wiki.gentoo.org/wiki/$1') - self.assertSite('http://wiki.arabeyes.org/$1') - self.assertSite('http://tfwiki.net/wiki/$1') - self.assertAllPass() + def test_wikinvest(self): + """Test detection of MediaWiki sites for www.wikinvest.com.""" + self.assertNoSite('http://www.wikinvest.com/$1') # v1.9 - @unittest.expectedFailure # Test no longer valid - def test_detect_failure(self): - """Test detection failure for MediaWiki sites with an API.""" - # SSL certificate verification fails - self.assertNoSite('http://hackerspaces.org/wiki/$1') - self.assertAllError() + +class PreAPISiteTestCase(SiteDetectionTestCase): + + """Test detection of MediaWiki sites prior to the API.""" + + def test_wikif1(self): + """Test detection of MediaWiki sites for www.wikif1.org.""" + self.assertNoSite('http://www.wikif1.org/$1') + + def test_thelemapedia(self): + """Test detection of MediaWiki sites for www.thelemapedia.org.""" + self.assertNoSite('http://www.thelemapedia.org/index.php/$1') + + def test_blahus(self): + """Test detection of MediaWiki sites for esperanto.blahus.cz.""" + self.assertNoSite('http://esperanto.blahus.cz/cxej/vikio/index.php/$1') + + def test_werelate(self): + """Test detection of MediaWiki sites for www.werelate.org.""" + self.assertNoSite('http://www.werelate.org/wiki/$1') + + def test_otterstedt(self): + """Test detection of MediaWiki sites for www.otterstedt.de.""" + self.assertNoSite('http://www.otterstedt.de/wiki/index.php/$1') + + def test_mozillazine(self): + """Test detection of MediaWiki sites for kb.mozillazine.org.""" + self.assertNoSite('http://kb.mozillazine.org/$1') + + +class APIHiddenTestCase(SiteDetectionTestCase): + + """Test MediaWiki sites with a hidden enabled API.""" + + def test_wikisophia(self): + """Test wikisophia.org which has redirect problems. + + /index.php?title=$1 reports 404, however a wiki exists there, + but the API is also hidden. + """ + self.assertNoSite('http://wikisophia.org/index.php?title=$1') @unittest.expectedFailure - def test_api_hidden(self): - """Test MediaWiki sites with a hidden enabled API.""" - # api.php is not available - self.assertNoSite('http://wiki.animutationportal.com/index.php/$1') - # HTML looks like it has an API, but redirect rules prevent access + def test_ecoreality(self): + """Test detection of MediaWiki sites for www.EcoReality.org. + + api.php is not available. HTML looks like it has an API, but redirect + rules prevent access. + """ self.assertNoSite('http://www.EcoReality.org/wiki/$1') - self.assertAllError() - def test_api_disabled(self): - """Test MediaWiki sites without an enabled API.""" + +class FailingSiteTestCase(SiteDetectionTestCase): + + """Test detection failure for MediaWiki sites with an API.""" + + def test_animutationportal(self): + """Test detection of MediaWiki sites for wiki.animutationportal.com. + + SSL certificate verification fails. + """ + self.assertNoSite('http://wiki.animutationportal.com/index.php/$1') + + @unittest.expectedFailure + def test_hackerspaces(self): + """Test detection of MediaWiki sites for hackerspaces.org.""" + self.assertNoSite('http://hackerspaces.org/wiki/$1') + + +class APIDisabledTestCase(SiteDetectionTestCase): + + """Test MediaWiki sites without an enabled API.""" + + def test_linuxquestions(self): + """Test detection of MediaWiki sites for wiki.linuxquestions.org.""" self.assertNoSite('http://wiki.linuxquestions.org/wiki/$1') - self.assertAllError() - def test_offline_sites(self): - """Test offline sites.""" - self.assertNoSite('http://seattlewiki.org/wiki/$1') - self.assertAllError() - def test_pre_api_sites(self): - """Test detection of MediaWiki sites prior to the API.""" - self.assertNoSite('http://www.wikif1.org/$1') - self.assertNoSite('http://www.thelemapedia.org/index.php/$1') - self.assertNoSite('http://esperanto.blahus.cz/cxej/vikio/index.php/$1') - self.assertNoSite('http://www.werelate.org/wiki/$1') - self.assertNoSite('http://www.otterstedt.de/wiki/index.php/$1') - self.assertNoSite('http://kb.mozillazine.org/$1') - self.assertAllError() +class NoSiteTestCase(SiteDetectionTestCase): - def test_detect_nosite(self): - """Test detection of non-wiki sites.""" + """Test detection of non-wiki sites.""" + + def test_imdb(self): + """Test detection of MediaWiki sites for www.imdb.com.""" self.assertNoSite('http://www.imdb.com/name/nm$1/') - self.assertNoSite('http://www.ecyrd.com/JSPWiki/Wiki.jsp?page=$1') - self.assertNoSite('http://operawiki.info/$1') - self.assertNoSite( - 'http://www.tvtropes.org/pmwiki/pmwiki.php/Main/$1') - self.assertNoSite('http://c2.com/cgi/wiki?$1') - self.assertNoSite('https://phabricator.wikimedia.org/$1') - self.assertNoSite( - 'http://www.merriam-webster.com/cgi-bin/dictionary?book=Dictionary&va=$1') - self.assertNoSite('http://arxiv.org/abs/$1') - self.assertAllError() - def test_musicbrainz_doc(self): - """Test http://musicbrainz.org/doc/ which has a page 'api.php'.""" - # Possible false positive caused by the existance of a page - # called http://musicbrainz.org/doc/api.php + def test_ecyrd(self): + """Test detection of MediaWiki sites for www.ecyrd.com.""" + self.assertNoSite('http://www.ecyrd.com/JSPWiki/Wiki.jsp?page=$1') + + def test_operawiki(self): + """Test detection of MediaWiki sites for operawiki.info.""" + self.assertNoSite('http://operawiki.info/$1') + + def test_tvtropes(self): + """Test detection of MediaWiki sites for www.tvtropes.org.""" + self.assertNoSite('http://www.tvtropes.org/pmwiki/pmwiki.php/Main/$1') + + def test_c2(self): + """Test detection of MediaWiki sites for c2.com.""" + self.assertNoSite('http://c2.com/cgi/wiki?$1') + + def test_phabricator(self): + """Test detection of MediaWiki sites for phabricator.wikimedia.org.""" + self.assertNoSite('https://phabricator.wikimedia.org/$1') + + def test_merriam_webster(self): + """Test detection of MediaWiki sites for www.merriam-webster.com.""" + self.assertNoSite( + 'http://www.merriam-webster.com/' + 'cgi-bin/dictionary?book=Dictionary&va=$1') + + def test_arxiv(self): + """Test detection of MediaWiki sites for arxiv.org.""" + self.assertNoSite('http://arxiv.org/abs/$1') + + +class OfflineSiteTestCase(SiteDetectionTestCase): + + """Test offline sites.""" + + def test_seattlewiki(self): + """Test detection of MediaWiki sites for seattlewiki.org.""" + self.assertNoSite('http://seattlewiki.org/wiki/$1') + + +class OtherSiteTestCase(SiteDetectionTestCase): + + """Test other non-MediaWiki sites.""" + + def test_musicbrainz(self): + """Test http://musicbrainz.org/doc/ which has a page 'api.php'. + + Possible false positive caused by the existance of a page called + http://musicbrainz.org/doc/api.php. + """ self.assertNoSite('http://musicbrainz.org/doc/$1') - self.assertAllError() if __name__ == '__main__': # pragma: no cover -- To view, visit https://gerrit.wikimedia.org/r/326358 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I170d0f136e8f0983aa82eb51bea6d28fbec50343 Gerrit-PatchSet: 6 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Magul <tomasz.magulski(a)gmail.com> Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Lokal Profil <lokal.profil(a)gmail.com> Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com> Gerrit-Reviewer: Maverick <manpreetkaur9411(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Sn1per <geofbot(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

7 years, 4 months

[Gerrit] pywikibot/core[master]: site.py: Adding new warnings 'nochange' and 'duplicateversions'

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/326923 ) Change subject: site.py: Adding new warnings 'nochange' and 'duplicateversions' ...................................................................... site.py: Adding new warnings 'nochange' and 'duplicateversions' It appears only when the uploaded file is duplicate with the current version or old versions of the file under the same filename. The messages are from MediaWiki i18n/en.json Bug: T153060 Change-Id: I6008bf855d4cbddabcdbd30d9cc9cafd7e87f7c5 --- M pywikibot/site.py 1 file changed, 6 insertions(+), 0 deletions(-) Approvals: Magul: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/site.py b/pywikibot/site.py index 5db67aa..92245fd 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -5930,6 +5930,12 @@ '"%(msg)s".', 'bad-prefix': 'Target filename has a bad prefix %(msg)s.', 'page-exists': 'Target filename exists but with a different file %(msg)s.', + + # API-returned message string will be timestamps, not much use here + 'nochange': 'The upload is an exact duplicate of the current version of ' + 'this file.', + 'duplicateversions': 'The upload is an exact duplicate of older ' + 'version(s) of this file.', } # An offset != 0 doesn't make sense without a file key -- To view, visit https://gerrit.wikimedia.org/r/326923 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I6008bf855d4cbddabcdbd30d9cc9cafd7e87f7c5 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Zhuyifei1999 <zhuyifei1999(a)gmail.com> Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

7 years, 4 months

[Gerrit] pywikibot/core[master]: Fix for test_recentchanges_timespan

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/324241 ) Change subject: Fix for test_recentchanges_timespan ...................................................................... Fix for test_recentchanges_timespan Adjust test_recentchanges_timespan for change Ia636aef46dd36b937a78cb71c9b98819753be2ff Bug: T151856 Change-Id: I043d1f39cb854aad9933a9f28f3466ff79a64a94 --- M tests/pagegenerators_tests.py 1 file changed, 5 insertions(+), 1 deletion(-) Approvals: Dalba: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py index c97cc4e..50d0122 100755 --- a/tests/pagegenerators_tests.py +++ b/tests/pagegenerators_tests.py @@ -854,7 +854,11 @@ gen = gf.getCombinedGenerator() self.assertIsNotNone(gen) self.assertRaises(ValueError, gf.handleArg, '-recentchanges:3,2,1') - self.assertRaises(ValueError, gf.handleArg, '-recentchanges:-10,20') + self.assertRaises(ValueError, gf.handleArg, '-recentchanges:12,-12') + self.assertRaises( + ValueError, gf.handleArg, '-recentchanges:visualeditor,3,2,1') + self.assertRaises( + ValueError, gf.handleArg, '-recentchanges:"mobile edit,-10,20"') def test_recentchanges_default(self): """Test recentchanges generator with default setting.""" -- To view, visit https://gerrit.wikimedia.org/r/324241 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I043d1f39cb854aad9933a9f28f3466ff79a64a94 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Magul <tomasz.magulski(a)gmail.com> Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Lokal Profil <lokal.profil(a)gmail.com> Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

7 years, 4 months

[Gerrit] pywikibot/core[master]: Fix generating WMF Labs families

by jenkins-bot (Code Review)

jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/323098 ) Change subject: Fix generating WMF Labs families ...................................................................... Fix generating WMF Labs families Also creation of family with interwikis on Travis. Bug: T151372 Change-Id: Iefe1ccee01468586706780796c80e267c1a3b543 --- M .travis.yml M generate_family_file.py 2 files changed, 14 insertions(+), 15 deletions(-) Approvals: Dalba: Looks good to me, approved jenkins-bot: Verified diff --git a/.travis.yml b/.travis.yml index 9d46272..ca5e14b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -61,10 +61,10 @@ - python pwb.py generate_family_file 'https://wiki.musicbrainz.org/' musicbrainz 'n' - if [[ $FAMILY == 'wpbeta' ]]; then - python -m generate_family_file 'http://'$LANGUAGE'.wikipedia.beta.wmflabs.org/' 'wpbeta' 'n' ; + python -m generate_family_file 'http://'$LANGUAGE'.wikipedia.beta.wmflabs.org/' 'wpbeta' 'y' ; fi - if [[ $FAMILY == 'wsbeta' ]]; then - python -m generate_family_file 'http://'$LANGUAGE'.wikisource.beta.wmflabs.org/' 'wsbeta' 'n' ; + python -m generate_family_file 'http://'$LANGUAGE'.wikisource.beta.wmflabs.org/' 'wsbeta' 'y' ; fi - python -W error::UserWarning -m generate_user_files -dir:~/.pywikibot/ -family:$FAMILY -lang:$LANGUAGE -v -user:"$PYWIKIBOT2_USERNAME" diff --git a/generate_family_file.py b/generate_family_file.py index 5505b69..cd1caa7 100755 --- a/generate_family_file.py +++ b/generate_family_file.py @@ -64,7 +64,7 @@ print("Generating family file from %s" % self.base_url) w = Wiki(self.base_url) - self.wikis[w.iwpath] = w + self.wikis[w.lang] = w print() print("==================================") print("api url: %s" % w.api) @@ -122,9 +122,9 @@ print("Loading wikis... ") for lang in self.langs: print(" * %s... " % (lang[u'prefix']), end="") - if lang[u'url'] not in self.wikis: + if lang['prefix'] not in self.wikis: try: - self.wikis[lang[u'url']] = Wiki(lang[u'url']) + self.wikis[lang['prefix']] = Wiki(lang['url']) print("downloaded") except Exception as e: print(e) @@ -167,38 +167,37 @@ self.langs = { """.lstrip() % {'url': self.base_url, 'name': self.name}) - for w in self.wikis.values(): + for k, w in self.wikis.items(): f.write(" '%(lang)s': '%(hostname)s',\n" - % {'lang': w.lang, 'hostname': urlparse(w.server).netloc}) + % {'lang': k, 'hostname': urlparse(w.server).netloc}) f.write(" }\n\n") - f.write(" def scriptpath(self, code):\n") f.write(" return {\n") - for w in self.wikis.values(): + for k, w in self.wikis.items(): f.write(" '%(lang)s': '%(path)s',\n" - % {'lang': w.lang, 'path': w.scriptpath}) + % {'lang': k, 'path': w.scriptpath}) f.write(" }[code]\n") f.write("\n") f.write(" @deprecated('APISite.version()')\n") f.write(" def version(self, code):\n") f.write(" return {\n") - for w in self.wikis.values(): + for k, w in self.wikis.items(): if w.version is None: - f.write(" '%(lang)s': None,\n" % {'lang': w.lang}) + f.write(" '%(lang)s': None,\n" % {'lang': k}) else: f.write(" '%(lang)s': u'%(ver)s',\n" - % {'lang': w.lang, 'ver': w.version}) + % {'lang': k, 'ver': w.version}) f.write(" }[code]\n") f.write("\n") f.write(" def protocol(self, code):\n") f.write(" return {\n") - for w in self.wikis.values(): + for k, w in self.wikis.items(): f.write(" '%(lang)s': u'%(protocol)s',\n" - % {'lang': w.lang, 'protocol': urlparse(w.server).scheme}) + % {'lang': k, 'protocol': urlparse(w.server).scheme}) f.write(" }[code]\n") -- To view, visit https://gerrit.wikimedia.org/r/323098 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Iefe1ccee01468586706780796c80e267c1a3b543 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Magul <tomasz.magulski(a)gmail.com> Gerrit-Reviewer: Dalba <dalba.wiki(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Lokal Profil <lokal.profil(a)gmail.com> Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

7 years, 4 months

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

Pywikibot-commits December 2016