jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/680301 )

Change subject: [parser] Make mwparserfromhell or wikitextparser mandatory ......................................................................

[parser] Make mwparserfromhell or wikitextparser mandatory

- check whether mwparserfromhell or wikitextparser is installed and raise ImportError if no package is available - update tox.ini - update some documentation hints

Bug: T106763 Change-Id: Ie89d08074a9d9b6380a0276fac261b8f6a55156f --- M ROADMAP.rst M pywikibot/textlib.py M tests/__init__.py M tox.ini 4 files changed, 20 insertions(+), 60 deletions(-)

Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified

diff --git a/ROADMAP.rst b/ROADMAP.rst index 36e8705..b242ebb 100644 --- a/ROADMAP.rst +++ b/ROADMAP.rst @@ -1,7 +1,7 @@ Current release changes ^^^^^^^^^^^^^^^^^^^^^^^

-* (No changes yet) +* mwparserfromhell or wikitextparser MediaWiki markup parser is mandatory (T106763)

Deprecations ^^^^^^^^^^^^ @@ -14,7 +14,6 @@ * 6.2.0: empty_iterator will be removed in favour of iter() * 6.1.0: tools.frozenmap will be removed in favour of types.MappingProxyType * 6.1.0: tools.DotReadableDict will be removed -* 6.1.0: mwparserfromhell or wikitextparser MediaWiki markup parser becomes mandatory (T106763) * 6.1.0: textlib.unescape() function will be removed in favour of html.unescape() * 6.0.1: Site.undeletepage() and Site.undelete_file_versions() will be removed in favour of Site.undelete() method * 6.0.1: Site.deletepage() and Site.deleteoldimage() will be removed in favour of Site.delete() method diff --git a/pywikibot/textlib.py b/pywikibot/textlib.py index 2b14c97..696c6c3 100644 --- a/pywikibot/textlib.py +++ b/pywikibot/textlib.py @@ -37,8 +37,18 @@ except ImportError: try: import mwparserfromhell as wikitextparser - except ImportError as e: - wikitextparser = e + except ImportError: + # print required because pywikibot is not imported completely + raise ImportError(""" +Pywikibot is missing a MediaWiki markup parser which is necessary. +Please update the required module with either + + pip install "mwparserfromhell>=0.5.0" + +or + + pip install "wikitextparser>=0.47.0" +""") from None

ETPType = List[Tuple[str, OrderedDictType[str, str]]]

@@ -1585,16 +1595,11 @@ only the last value provided will be returned.

This uses the package L{mwparserfromhell} or L{wikitextparser} as - MediaWiki markup parser. Otherwise it falls back on a regex based - implementation but it becomes mandatory that one of them is + MediaWiki markup parser. It is mandatory that one of them is installed.

There are minor differences between the two implementations.

- The two implementations return nested templates in a different - order, i.e. for `{{a|b={{c}}}}`, parsers returns `[a, c]`, whereas - regex returns `[c, a]`. - The parser packages preserves whitespace in parameter names and values.

@@ -1613,45 +1618,6 @@ *New in version 6.1:* *wikitextparser* package is supported; either *wikitextparser* or *mwparserfromhell* is strictly recommended. """ - use_regex = isinstance(wikitextparser, ImportError) - - if remove_disabled_parts: - text = removeDisabledParts(text) - - if use_regex: - issue_deprecation_warning(""" -Pywikibot needs a MediaWiki markup parser. -Please install the requested module with either - - pip install "mwparserfromhell>=0.5.0" - -or - - pip install "wikitextparser>=0.47.0" - -Using pywikibot without MediaWiki markup parser""", - warning_class=FutureWarning, - since='20210416') - - return _extract_templates_and_params_regex(text, False, strip) - return _extract_templates_and_params_parser(text, strip) - - -def _extract_templates_and_params_parser(text: str, - strip: bool = False) -> ETPType: - """ - Extract templates with params using mwparserfromhell. - - This function should not be called directly. - - Use extract_templates_and_params, which will select this parser - implementation if the mwparserfromhell or wikitextparser package is - installed. - - @param text: The wikitext from which templates are extracted - @param strip: if enabled, strip arguments and values of templates - @return: list of template name and params - """ def explicit(param): try: attr = param.showkey @@ -1659,6 +1625,9 @@ attr = not param.positional return attr

+ if remove_disabled_parts: + text = removeDisabledParts(text) + parser_name = wikitextparser.__name__ pywikibot.log('Using {!r} wikitext parser'.format(parser_name))

@@ -1701,7 +1670,7 @@ global wikitextparser saved_parser = wikitextparser import mwparserfromhell as wikitextparser - result = _extract_templates_and_params_parser(text, strip) + result = extract_templates_and_params(text, strip=strip) wikitextparser = saved_parser return result

@@ -1723,14 +1692,6 @@ @param strip: if enabled, strip arguments and values of templates @return: list of template name and params """ - return _extract_templates_and_params_regex(text, remove_disabled_parts, - strip) - - -def _extract_templates_and_params_regex(text: str, - remove_disabled_parts: bool = True, - strip: bool = True) -> ETPType: - """DEPRECATED. Extract templates with params using a regex.""" # remove commented-out stuff etc. if remove_disabled_parts: thistxt = removeDisabledParts(text) diff --git a/tests/__init__.py b/tests/__init__.py index ebac252..ce88efe 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -23,7 +23,7 @@ # Verify that the unit tests have a base working environment: # - requests is mandatory # however if unavailable this will fail on use; see pywikibot/tools.py -# - mwparserfromhell or wikitextparser should be used but the dependency +# - mwparserfromhell or wikitextparser is mandatory but the dependency # is checked by textlib already import requests # noqa: F401

diff --git a/tox.ini b/tox.ini index 7bf0112..98c63f2 100644 --- a/tox.ini +++ b/tox.ini @@ -38,10 +38,10 @@ fasttest: pytest-attrib>=0.1.3 fasttest: pytest-subtests >= 0.3.2 fasttest: mock + fasttest: .[mwparserfromhell] fasttest: .[scripts]

fasttest-py35: .[html] - fasttest-py35: .[mwparserfromhell] fasttest-py37: .[wikitextparser]

deeptest: .[html]