jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/1033691?usp=email )

Change subject: [Fix] Add namespaces parameter to BasePage.templates and itertemplates ......................................................................

[Fix] Add namespaces parameter to BasePage.templates and itertemplates

- add namespaces parameter to BasePage.templates() and require keyworded arguments. Retrieve all pages from API:Templates and filter namespace on return. Update documentation. - add namespaces parameter to BasePage.itertemplates() and require keyworded arguments except for total (otherwise the positional arguments cannot be deprecated if the total will be the last argument later like in APISite.pagetemplates(). - only use pages in TEMPLATE namespaces within BasePage.isDisambig() because the other pages are also in that namespace. Use tempates() instead of itertemplates() here to cache. Use site method disambig() instead of Family.disambig(). Update documentations. - Update documentation in APISite.pagetemplates(), reorder keyword parameters which are keyword only. - use TEMPLATE namespace to get templates in Page.templatesWithParams because the namespace of the links to be compared is TEMPLATE too. Use itertemplates here because the result can be cached by the calling method. This solves T365199. - use TEMPLATE namespace to get templates in commonscat.py and solve_disambiguation.py scripts because "real" templates are requuired here. - update utils.DryParamInfo class for pg tests

Bug: T365199 Change-Id: I0e82e889039f3e9cbf36f5faf038331d8a17d3fe --- M docs/api_ref/pywikibot.site.rst M pywikibot/family.py M pywikibot/page/_basepage.py M pywikibot/page/_page.py M pywikibot/site/_generators.py M scripts/commonscat.py M scripts/solve_disambiguation.py M tests/utils.py 8 files changed, 129 insertions(+), 58 deletions(-)

Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified

diff --git a/docs/api_ref/pywikibot.site.rst b/docs/api_ref/pywikibot.site.rst index 1380d73..347430b 100644 --- a/docs/api_ref/pywikibot.site.rst +++ b/docs/api_ref/pywikibot.site.rst @@ -61,12 +61,15 @@ :rtype: tuple[str, ...]

- .. method:: disambig(fallback: str = '_default') + .. method:: disambig(fallback = '_default')

Return list of disambiguation templates.

.. seealso:: :meth:`family.Family.disambig`

+ :param str | None fallback: + :rtype: list[str] +

.. method:: protocol()

diff --git a/pywikibot/family.py b/pywikibot/family.py index 79042e7..22961fd 100644 --- a/pywikibot/family.py +++ b/pywikibot/family.py @@ -453,8 +453,11 @@ """ return self.archived_page_templates.get(code, ())

- def disambig(self, code, fallback: str = '_default'): - """Return list of disambiguation templates.""" + def disambig(self, code, fallback: str | None = '_default') -> list[str]: + """Return list of disambiguation templates. + + :raises KeyError: unknown title for disambig template + """ if code in self.disambiguationTemplates: return self.disambiguationTemplates[code]

diff --git a/pywikibot/page/_basepage.py b/pywikibot/page/_basepage.py index 61623b6..de45151 100644 --- a/pywikibot/page/_basepage.py +++ b/pywikibot/page/_basepage.py @@ -39,6 +39,7 @@ ComparableMixin, cached, deprecated, + deprecate_positionals, first_upper, issue_deprecation_warning, remove_last_args, @@ -900,18 +901,19 @@ return self.namespace() == Namespace.FILE

def isDisambig(self) -> bool: - """ - Return True if this is a disambiguation page, False otherwise. + """Return True if this is a disambiguation page, False otherwise.

By default, it uses the Disambiguator extension's result. The - identification relies on the presence of the __DISAMBIG__ magic word - which may also be transcluded. + identification relies on the presence of the ``__DISAMBIG__`` + magic word which may also be transcluded.

- If the Disambiguator extension isn't activated for the given site, - the identification relies on the presence of specific templates. - First load a list of template names from the Family file; - if the value in the Family file is None or no entry was made, look for - the list on [[MediaWiki:Disambiguationspage]]. If this page does not + If the Disambiguator extension isn't activated for the given + site, the identification relies on the presence of specific + templates. First load a list of template names from the + :class:`Family<family.Family>` file via :meth:`BaseSite.disambig() + <pywikibot.site._basesite.BaseSite.disambig>`; if the value in + the Family file not found, look for the list on + ``[[MediaWiki:Disambiguationspage]]``. If this page does not exist, take the MediaWiki message. 'Template:Disambig' is always assumed to be default, and will be appended regardless of its existence. @@ -925,19 +927,25 @@ default = set(self.site.family.disambig('_default')) except KeyError: default = {'Disambig'} + try: - distl = self.site.family.disambig(self.site.code, - fallback=False) + distl = self.site.disambig(fallback=False) except KeyError: distl = None - if distl is None: + + if distl: + # Normalize template capitalization + self.site._disambigtemplates = {first_upper(t) for t in distl} + else: + # look for the list on [[MediaWiki:Disambiguationspage]] disambigpages = pywikibot.Page(self.site, 'MediaWiki:Disambiguationspage') if disambigpages.exists(): disambigs = {link.title(with_ns=False) for link in disambigpages.linkedPages() - if link.namespace() == 10} + if link.namespace() == Namespace.TEMPLATE} elif self.site.has_mediawiki_message('disambiguationspage'): + # take the MediaWiki message message = self.site.mediawiki_message( 'disambiguationspage').split(':', 1)[1] # add the default template(s) for default mw message @@ -946,16 +954,16 @@ else: disambigs = default self.site._disambigtemplates = disambigs - else: - # Normalize template capitalization - self.site._disambigtemplates = {first_upper(t) for t in distl} - templates = {tl.title(with_ns=False) for tl in self.templates()} + + templates = {tl.title(with_ns=False) + for tl in self.templates(namespaces=Namespace.TEMPLATE)} disambigs = set() # always use cached disambig templates disambigs.update(self.site._disambigtemplates) # see if any template on this page is in the set of disambigs disambig_in_page = disambigs.intersection(templates) - return self.namespace() != 10 and bool(disambig_in_page) + return (self.namespace() != Namespace.TEMPLATE + and bool(disambig_in_page))

def getReferences(self, follow_redirects: bool = True, @@ -1601,17 +1609,33 @@ """Convenience function to get the Wikibase item of a page.""" return pywikibot.ItemPage.fromPage(self)

- def templates(self, content: bool = False) -> list[pywikibot.Page]: - """ - Return a list of Page objects for templates used on this Page. + @deprecate_positionals(since='9.2') + def templates(self, + *, + content: bool = False, + namespaces: NamespaceArgType = None) -> list[pywikibot.Page]: + """Return a list of Page objects for templates used on this Page.

- Template parameters are ignored. This method only returns embedded - templates, not template pages that happen to be referenced through - a normal link. + This method returns a list of pages which are embedded as + templates even they are not in the TEMPLATE: namespace. This + method caches the result. If *namespaces* is used, all pages are + retrieved and cached but the result is filtered. + + .. versionchanged:: 2.0 + a list of :class:`pywikibot.Page` is returned instead of a + list of template titles. The given pages may have namespaces + different from TEMPLATE namespace. *get_redirect* parameter + was removed. + .. versionchanged:: 9.2 + *namespaces* parameter was added; all parameters must be given + as keyword arguments. + + .. seealso:: + - :meth:`itertemplates`

:param content: if True, retrieve the content of the current version of each template (default False) - :param content: bool + :param namespaces: Only iterate pages in these namespaces """ # Data might have been preloaded # Delete cache if content is needed and elements have no content @@ -1620,32 +1644,53 @@ and not all(t.has_content() for t in self._templates)): del self._templates

+ # retrieve all pages in _templates and filter namespaces later if not hasattr(self, '_templates'): self._templates = set(self.itertemplates(content=content))

+ if namespaces is not None: + ns = self.site.namespaces.resolve(namespaces) + return [t for t in self._templates if t.namespace() in ns] + return list(self._templates)

+ @deprecate_positionals(since='9.2') def itertemplates( self, total: int | None = None, + *, content: bool = False, + namespaces: NamespaceArgType = None ) -> Iterable[pywikibot.Page]: - """ - Iterate Page objects for templates used on this Page. + """Iterate Page objects for templates used on this Page.

- Template parameters are ignored. This method only returns embedded - templates, not template pages that happen to be referenced through - a normal link. + This method yield pages embedded as templates even they are not + in the TEMPLATE: namespace. The retrieved pages are not cached + but they can be yielded from the cache of a previous + :meth:`templates` call. + + .. versionadded:: 2.0 + .. versionchanged:: 9.2 + *namespaces* parameter was added; all parameters except + *total* must be given as keyword arguments. + + .. seealso:: + - :meth:`site.APISite.pagetemplates() + <pywikibot.site._generators.GeneratorsMixin.pagetemplates>` + - :meth:`templates` + - :meth:`getReferences`

:param total: iterate no more than this number of pages in total :param content: if True, retrieve the content of the current version of each template (default False) - :param content: bool + :param namespaces: Only iterate pages in these namespaces """ if hasattr(self, '_templates'): - return itertools.islice(self.templates(content=content), total) + return itertools.islice(self.templates( + content=content, namespaces=namespaces), total)

- return self.site.pagetemplates(self, total=total, content=content) + return self.site.pagetemplates( + self, content=content, namespaces=namespaces, total=total)

def imagelinks( self, diff --git a/pywikibot/page/_page.py b/pywikibot/page/_page.py index 60573f6..8dcda60 100644 --- a/pywikibot/page/_page.py +++ b/pywikibot/page/_page.py @@ -9,7 +9,7 @@ itself, including its contents. """ # -# (C) Pywikibot team, 2008-2023 +# (C) Pywikibot team, 2008-2024 # # Distributed under the terms of the MIT license. # @@ -27,6 +27,7 @@ ) from pywikibot.page._basepage import BasePage from pywikibot.page._toolforge import WikiBlameMixin +from pywikibot.site import Namespace from pywikibot.tools import cached

@@ -58,7 +59,9 @@ """ return textlib.extract_templates_and_params(self.text, True, True)

- def templatesWithParams(self): # noqa: N802 + def templatesWithParams( # noqa: N802 + self, + ) -> list[tuple[pywikibot.Page, list[str]]]: """Return templates used on this Page.

The templates are extracted by :meth:`raw_extracted_templates`, @@ -68,14 +71,14 @@ All parameter keys and values for each template are stripped of whitespace.

- :return: a list of tuples with one tuple for each template invocation - in the page, with the template Page as the first entry and a list - of parameters as the second entry. - :rtype: list of (pywikibot.page.Page, list) + :return: a list of tuples with one tuple for each template + invocation in the page, with the template Page as the first + entry and a list of parameters as the second entry. """ # WARNING: may not return all templates used in particularly # intricate cases such as template substitution - titles = {t.title() for t in self.templates()} + titles = {t.title() + for t in self.itertemplates(namespaces=Namespace.TEMPLATE)} templates = self.raw_extracted_templates # backwards-compatibility: convert the dict returned as the second # element into a list in the format used by old scripts @@ -83,7 +86,7 @@ for template in templates: try: link = pywikibot.Link(template[0], self.site, - default_namespace=10) + default_namespace=Namespace.TEMPLATE) if link.canonical_title() not in titles: continue except Error: diff --git a/pywikibot/site/_generators.py b/pywikibot/site/_generators.py index 1ee5a29..af2aa43 100644 --- a/pywikibot/site/_generators.py +++ b/pywikibot/site/_generators.py @@ -504,21 +504,29 @@ self, page: pywikibot.Page, *, + content: bool = False, namespaces: NamespaceArgType = None, total: int | None = None, - content: bool = False, ) -> Iterable[pywikibot.Page]: - """Iterate templates transcluded (not just linked) on the page. + """Iterate pages transcluded (not just linked) on the page.

- .. seealso:: :api:`Templates` + .. note: You should not use this method directly; use + :meth:`pywikibot.Page.itertemplates` instead.

+ .. seealso:: + - :api:`Templates` + - :meth:`page.BasePage.itertemplates` + + :param content: if True, load the current content of each + iterated page (default False) :param namespaces: Only iterate pages in these namespaces - :param content: if True, load the current content of each iterated page - (default False) + :param total: maximum number of pages to retrieve in total

:raises KeyError: a namespace identifier was not resolved :raises TypeError: a namespace identifier has an inappropriate type such as NoneType or bool + :raises UnsupportedPageError: a Page object is not supported due + to namespace restriction """ tltitle = page.title(with_section=False).encode(self.encoding()) return self._generator(api.PageGenerator, type_arg='templates', diff --git a/scripts/commonscat.py b/scripts/commonscat.py index e1f3191..bbfc466 100755 --- a/scripts/commonscat.py +++ b/scripts/commonscat.py @@ -38,7 +38,7 @@ # *Found one template. Add this template # *Found more templates. Ask the user <- still have to implement this # -# (C) Pywikibot team, 2008-2023 +# (C) Pywikibot team, 2008-2024 # # Distributed under the terms of the MIT license. # @@ -50,6 +50,7 @@ from pywikibot import i18n, pagegenerators from pywikibot.bot import ConfigParserBot, ExistingPageBot from pywikibot.exceptions import InvalidTitleError +from pywikibot.site import Namespace from pywikibot.textlib import add_text

@@ -264,8 +265,9 @@

for template in templates_to_ignore: if not isinstance(template, tuple): - for pageTemplate in page.templates(): - if pageTemplate.title(with_ns=False) == template: + for page_template in page.itertemplates( + namespaces=Namespace.TEMPLATE): + if page_template.title(with_ns=False) == template: return True else: for inPageTemplate, params in page.templatesWithParams(): @@ -512,7 +514,7 @@ commonsPage.getRedirectTarget().title(with_ns=False))

if (pywikibot.Page(commonsPage.site, 'Template:Category redirect') - in commonsPage.templates()): + in commonsPage.itertemplates(namespaces=Namespace.TEMPLATE)): pywikibot.log( 'getCommonscat: The category is a category redirect') for template, param in commonsPage.templatesWithParams(): diff --git a/scripts/solve_disambiguation.py b/scripts/solve_disambiguation.py index 26c1dae..878e048 100755 --- a/scripts/solve_disambiguation.py +++ b/scripts/solve_disambiguation.py @@ -107,6 +107,7 @@ NoPageError, PageSaveRelatedError, ) +from pywikibot.site import Namespace from pywikibot.tools import first_lower, first_upper from pywikibot.tools.formatter import SequenceOutputter

@@ -1037,7 +1038,8 @@ if primary: primary_page = pywikibot.Page(page.site, 'Template:' + primary) - if primary and primary_page in page.templates(): + if primary and primary_page in page.itertemplates( + namespaces=Namespace.TEMPLATE): baseTerm = page.title() for template, params in page.templatesWithParams(): if params and template == primary_page: diff --git a/tests/utils.py b/tests/utils.py index 7ea74d1..209ac34 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -11,7 +11,7 @@ import sys import unittest import warnings -from contextlib import contextmanager +from contextlib import contextmanager, suppress from subprocess import PIPE, Popen, TimeoutExpired from typing import Any

@@ -239,10 +239,15 @@

def __getitem__(self, name): """Return dry data or a dummy parameter block.""" - try: + with suppress(KeyError): return super().__getitem__(name) - except KeyError: - return {'name': name, 'limit': None} + + result = {'name': name, 'prefix': '', 'limit': {}} + + if name == 'query+templates': + result['limit'] = {'max': 1} + + return result

class DummySiteinfo: