[Pywikibot-commits] [Gerrit] pywikibot/core[master]: Add support for Linter API in PageGenerators

jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/406411 ) Change subject: Add support for Linter API in PageGenerators ...................................................................... Add support for Linter API in PageGenerators Add: - site.linter_pages in site.py to generate pages with linter errors - add linter info to Page properties in api.py - add linter info to cached properties of Page in page.py - '-linter' options and similar in pagegenerators.py See https://www.mediawiki.org/wiki/Extension:Linter#API Bug: T185519 Change-Id: Ic99f52ceb8b85186cbdc2e5e7b3c3bf657bfb173 --- M pywikibot/data/api.py M pywikibot/page.py M pywikibot/pagegenerators.py M pywikibot/site.py M tests/pagegenerators_tests.py M tests/site_tests.py 6 files changed, 148 insertions(+), 4 deletions(-) Approvals: jenkins-bot: Verified Xqt: Looks good to me, approved diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index 5fac264..01137a8 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -3268,3 +3268,9 @@ if "flowinfo" in pagedict: page._flowinfo = pagedict['flowinfo']['flow'] + + if 'lintId' in pagedict: + page._lintinfo = pagedict + page._lintinfo.pop('pageid') + page._lintinfo.pop('title') + page._lintinfo.pop('ns') diff --git a/pywikibot/page.py b/pywikibot/page.py index 12d847b..e44f393 100644 --- a/pywikibot/page.py +++ b/pywikibot/page.py @@ -160,7 +160,7 @@ '_contentmodel', '_langlinks', '_isredir', '_coords', '_preloadedtext', '_timestamp', '_applicable_protections', '_flowinfo', '_quality', '_pageprops', '_revid', '_quality_text', - '_pageimage', '_item' + '_pageimage', '_item', '_lintinfo', ) def __init__(self, source, title=u"", ns=0): diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index e67b64e..ab4ed51 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -276,6 +276,19 @@ "-pageid:pageid1,pageid2,." or "-pageid:'pageid1|pageid2|..'" and supplied multiple times for multiple pages. +-linter Work on pages that contains lint errors. Extension Linter + must be available on the site. + -linter select all categories. + -linter:high, -linter:medium or -linter:low select all + categories for that prio. + Single categories can be selected with commas as in + -linter:cat1,cat2,cat3 + + Adding '/int' indentifies Lint ID to start querying from: + e.g. -linter:high/10000 + + -linter:show just shows available categories. + FILTER OPTIONS ============== @@ -312,9 +325,9 @@ -ns:not:2,3 -ns:not:Help,File - If used with -newpages/-random/-randomredirect generators, - -namespace/ns must be provided before - -newpages/-random/-randomredirect. + If used with -newpages/-random/-randomredirect/linter + generators, -namespace/ns must be provided before + -newpages/-random/-randomredirect/linter. If used with -recentchanges generator, efficiency is improved if -namespace is provided before -recentchanges. @@ -699,6 +712,41 @@ value = 'Image:' + value page = pywikibot.FilePage(self.site, value) gen = FileLinksGenerator(page) + elif arg == '-linter': + if not self.site.has_extension('Linter'): + raise UnknownExtension( + '-linter needs a site with Linter extension.') + cats = self.site.siteinfo.get('linter') # Get linter categories. + valid_cats = [c for _list in cats.values() for c in _list] + + cat, sep, lint_from = value.partition('/') + if not lint_from: + lint_from = None + + if cat == 'show': # Display categories of lint errors. + _i = ' ' * 4 + txt = 'Available categories of lint errors:\n' + for prio, _list in cats.items(): + txt += '{indent}{prio}\n'.format(indent=_i, prio=prio) + for c in _list: + txt += '{indent}{cat}\n'.format(indent=2 * _i, cat=c) + pywikibot.output('%s' % txt) + return True + + if not cat: + lint_cats = valid_cats + elif cat in ['low', 'medium', 'high']: + lint_cats = cats[cat] + else: + lint_cats = cat.split(',') + for lint_cat in lint_cats: + if lint_cat not in valid_cats: + raise ValueError('Invalid category of lint errors: %s' + % cat) + + gen = self.site.linter_pages(lint_categories='|'.join(lint_cats), + namespaces=self.namespaces, + lint_from=lint_from) elif arg == '-unusedfiles': gen = UnusedFilesGenerator(total=intNone(value), site=self.site) elif arg == '-lonelypages': diff --git a/pywikibot/site.py b/pywikibot/site.py index c174f1d..1eff176 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -6883,6 +6883,61 @@ comparison = data['compare']['*'] return comparison + @need_extension('Linter') + def linter_pages(self, lint_categories=None, total=None, + namespaces=None, pageids=None, lint_from=None): + """Return a generator to pages containing linter errors. + + @param lint_categories: categories of lint errors + @type lntcategories: an iterable that returns values (str), + or a pipe-separated string of values. + + @param total: if not None, yielding this many items in total + @type total: int + + @param namespaces: only iterate pages in these namespaces + @type namespaces: iterable of basestring or Namespace key, + or a single instance of those types. May be a '|' separated + list of namespace identifiers. + + @param pageids: only include lint errors from the specified pageids + @type pageids: an iterable that returns pageids (str or int), + or a comma- or pipe-separated string of pageids + (e.g. '945097,1483753, 956608' or '945097|483753|956608') + + @param lint_from: Lint ID to start querying from + @type lint_from: str representing digit or integer + + @return: pages with Linter errors. + @rtype: generator of Page + + """ + query = self._generator(api.ListGenerator, type_arg='linterrors', + total=total, # Will set lntlimit + namespaces=namespaces) + + if lint_categories: + if isinstance(lint_categories, basestring): + lint_categories = lint_categories.split('|') + lint_categories = [p.strip() for p in lint_categories] + query.request['lntcategories'] = '|'.join(lint_categories) + + if pageids: + if isinstance(pageids, basestring): + pageids = pageids.split('|') + pageids = [p.strip() for p in pageids] + # Validate pageids. + pageids = (str(int(p)) for p in pageids if int(p) > 0) + query.request['lntpageid'] = '|'.join(pageids) + + if lint_from: + query.request['lntfrom'] = int(lint_from) + + for pageitem in query: + page = pywikibot.Page(self, pageitem['title']) + api.update_page(page, pageitem) + yield page + # Thanks API calls @need_extension('Thanks') def thank_revision(self, revid, source=None): diff --git a/tests/pagegenerators_tests.py b/tests/pagegenerators_tests.py index 11e4d6b..107a68d 100755 --- a/tests/pagegenerators_tests.py +++ b/tests/pagegenerators_tests.py @@ -1177,6 +1177,26 @@ self.assertIsNotNone(gen2) self.assertEqual(list(gen1), list(gen2)) + def test_linter_generator_ns_valid_cat(self): + """Test generator of pages with lint errors.""" + gf = pagegenerators.GeneratorFactory(site=self.site) + gf.handleArg('-ns:1') + gf.handleArg('-limit:3') + gf.handleArg('-linter:obsolete-tag') + gen = gf.getCombinedGenerator() + self.assertIsNotNone(gen) + pages = list(gen) + self.assertLessEqual(len(pages), 5) + for page in pages: + self.assertIsInstance(page, pywikibot.Page) + self.assertEqual(page._lintinfo['category'], 'obsolete-tag') + self.assertPagesInNamespaces(pages, set([1, ])) + + def test_linter_generator_invalid_cat(self): + """Test generator of pages with lint errors.""" + gf = pagegenerators.GeneratorFactory(site=self.site) + self.assertRaises(ValueError, gf.handleArg, '-linter:dummy') + class TestFactoryGeneratorWikibase(WikidataTestCase): diff --git a/tests/site_tests.py b/tests/site_tests.py index 0acfa44..bd5ec92 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -1065,6 +1065,21 @@ self.assertLessEqual(len(tuple(upgen)), 3) +class TestLinterPages(DefaultSiteTestCase): + + """Test linter_pages methods.""" + + def test_linter_pages(self): + """Test the deprecated site.logpages() method.""" + le = list(self.site.linter_pages( + lint_categories='obsolete-tag|missing-end-tag', total=5)) + self.assertLessEqual(len(le), 5) + for entry in le: + self.assertIsInstance(entry, pywikibot.Page) + self.assertIn(entry._lintinfo['category'], + ['obsolete-tag', 'missing-end-tag']) + + class TestImageUsage(DefaultSiteTestCase): """Test cases for Site.imageusage method.""" -- To view, visit https://gerrit.wikimedia.org/r/406411 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ic99f52ceb8b85186cbdc2e5e7b3c3bf657bfb173 Gerrit-PatchSet: 6 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Mpaa &lt;mpaa.wiki(a)gmail.com&gt; Gerrit-Reviewer: Framawiki &lt;framawiki(a)tools.wmflabs.org&gt; Gerrit-Reviewer: John Vandenberg &lt;jayvdb(a)gmail.com&gt; Gerrit-Reviewer: Mpaa &lt;mpaa.wiki(a)gmail.com&gt; Gerrit-Reviewer: Xqt &lt;info(a)gno.de&gt; Gerrit-Reviewer: Zoranzoki21 &lt;zorandori4444(a)gmail.com&gt; Gerrit-Reviewer: jenkins-bot <>