Xqt has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/810472 ) Change subject: [IMPR] Only follow redirects in harvest_template.py if no wikibase item exists ...................................................................... [IMPR] Only follow redirects in harvest_template.py if no wikibase item exists This implements the proposal no. 2 of T311883 - Make _template_link_target a public staticmethod - Test for InvalidPageError with linked_page.exists() call - do not follow redirect in template_link_target if the redirect page has a wikibase item - solve multiple return statements issue - Add test for this new behaviour Bug: T311883 Change-Id: I87fe427009f9bbe5db2208d0ed850eb0d48bd505 --- M scripts/harvest_template.py M tests/__init__.py A tests/harvest_templates_tests.py 3 files changed, 74 insertions(+), 19 deletions(-) Approvals: jenkins-bot: Verified Xqt: Looks good to me, approved diff --git a/scripts/harvest_template.py b/scripts/harvest_template.py index c1df570..a225b8d 100755 --- a/scripts/harvest_template.py +++ b/scripts/harvest_template.py @@ -106,7 +106,11 @@ from pywikibot import textlib from pywikibot.backports import List from pywikibot.bot import ConfigParserBot, OptionHandler, WikidataBot -from pywikibot.exceptions import InvalidTitleError, NoPageError +from pywikibot.exceptions import ( + InvalidPageError, + InvalidTitleError, + NoPageError, +) willstop = False @@ -203,15 +207,23 @@ titles.append(temp.title(with_ns=False)) return titles - def _template_link_target(self, item, link_text - ) -> Optional[pywikibot.ItemPage]: + @staticmethod + def template_link_target(item: pywikibot.ItemPage, + link_text: str) -> Optional[pywikibot.ItemPage]: + """Find the ItemPage target for a given link text. + + .. versionchanged:: 7.4 + Only follow the redirect target if redirect page has no + wikibase item. + """ link = pywikibot.Link(link_text) linked_page = pywikibot.Page(link) try: exists = linked_page.exists() - except InvalidTitleError: - pywikibot.error('"{}" is not a valid title so it cannot be linked.' - ' Skipping.'.format(link_text)) + except (InvalidTitleError, InvalidPageError): + pywikibot.error('"{}" is not a valid title or the page itself is ' + 'invalid so it cannot be linked. Skipping.' + .format(link_text)) return None if not exists: @@ -219,23 +231,24 @@ 'Skipping.'.format(linked_page)) return None - if linked_page.isRedirectPage(): - linked_page = linked_page.getRedirectTarget() - - try: - linked_item = pywikibot.ItemPage.fromPage(linked_page) - except NoPageError: - linked_item = None + while True: + try: + linked_item = pywikibot.ItemPage.fromPage(linked_page) + except NoPageError: + if linked_page.isRedirectPage(): + linked_page = linked_page.getRedirectTarget() + continue + linked_item = None + break if not linked_item or not linked_item.exists(): pywikibot.output('{} does not have a wikidata item to link with. ' 'Skipping.'.format(linked_page)) - return None - - if linked_item.title() == item.title(): + linked_item = None + elif linked_item.title() == item.title(): pywikibot.output('{} links to itself. Skipping.' .format(linked_page)) - return None + linked_item = None return linked_item @@ -295,7 +308,7 @@ for match in pywikibot.link_regex.finditer(value): matched = True link_text = match.group(1) - linked_item = self._template_link_target( + linked_item = self.template_link_target( item, link_text) added = False if linked_item: @@ -321,7 +334,7 @@ .format(claim.getID(), field, value)) continue - linked_item = self._template_link_target(item, value) + linked_item = self.template_link_target(item, value) if not linked_item: continue diff --git a/tests/__init__.py b/tests/__init__.py index 39ce2b0..fa97e19 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -148,6 +148,7 @@ 'fixing_redirects', 'generate_family_file', 'generate_user_files', + 'harvest_templates', 'interwikidata', 'l10n', 'patrolbot', diff --git a/tests/harvest_templates_tests.py b/tests/harvest_templates_tests.py new file mode 100644 index 0000000..4ea7a6e --- /dev/null +++ b/tests/harvest_templates_tests.py @@ -0,0 +1,41 @@ +#!/usr/bin/python3 +"""Tests for scripts/harvest_template.py.""" +# +# (C) Pywikibot team, 2022 +# +# Distributed under the terms of the MIT license. +# +import unittest +from contextlib import suppress + +from pywikibot import ItemPage +from scripts.harvest_template import HarvestRobot + +from tests.aspects import ScriptMainTestCase + + +class TestHarvestRobot(ScriptMainTestCase): + + """Test HarvestRobot.""" + + family = 'wikipedia' + code = 'cs' + + def test_template_link_target(self): + """Test template_link_target static method.""" + tests = [ + ('Pes', 'Q144'), + ('Imaginární číslo', 'Q9165172'), + ('Sequana', 'Q472766'), + ] + for link, item in tests: + with self.subTest(link=link, item=item): + dummy_item = ItemPage(self.site.data_repository(), 'Q1') + target = HarvestRobot.template_link_target(dummy_item, link) + self.assertIsInstance(target, ItemPage) + self.assertEqual(target.title(), item) + + +if __name__ == '__main__': # pragma: no cover + with suppress(SystemExit): + unittest.main() -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/810472 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I87fe427009f9bbe5db2208d0ed850eb0d48bd505 Gerrit-Change-Number: 810472 Gerrit-PatchSet: 4 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: D3r1ck01 <xsavitar.wiki(a)aol.com> Gerrit-Reviewer: Fomafix <fomafix(a)googlemail.com> Gerrit-Reviewer: JAn Dudík <jan.dudik(a)gmail.com> Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged