Xqt has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/594478 )

Change subject: [cleanup] Always use mwparserfromhell if present ......................................................................

[cleanup] Always use mwparserfromhell if present

use_mwparserfromhell is True by default and mwparserfromhell will be used in that case if the package is installed.

The extract_templates_and_params_regex function is also available if mwparserfromhell should not be used in rare cases. Note that this fallback does not work for deeply nested templates (> 3).

Update tests

Change-Id: Id055618106eec13957c0038210cbabf19771ad5b --- M pywikibot/config2.py M pywikibot/textlib.py M tests/textlib_tests.py 3 files changed, 22 insertions(+), 37 deletions(-)

Approvals: Legoktm: Looks good to me, approved jenkins-bot: Verified

diff --git a/pywikibot/config2.py b/pywikibot/config2.py index b5e5a1c..641f5b2 100644 --- a/pywikibot/config2.py +++ b/pywikibot/config2.py @@ -110,10 +110,11 @@ # to other modules.

_private_values = {'authenticate', 'db_password'} -_deprecated_variables = {'available_ssl_project', 'fake_user_agent', - 'interwiki_contents_on_disk', 'line_separator', 'LS', - 'panoramio', 'proxy', 'special_page_limit', - 'sysopnames', 'use_SSL_onlogin', 'use_SSL_always'} +_deprecated_variables = { + 'available_ssl_project', 'fake_user_agent', 'interwiki_contents_on_disk', + 'line_separator', 'LS', 'panoramio', 'proxy', 'special_page_limit', + 'sysopnames', 'use_mwparserfromhell', 'use_SSL_onlogin', 'use_SSL_always', +}

# ############# ACCOUNT SETTINGS ##############

@@ -870,13 +871,6 @@ # processing. As higher this value this effect will decrease. max_queue_size = 64

-# Settings to enable mwparserfromhell -# https://mwparserfromhell.readthedocs.org/en/latest/ -# Currently used in textlib.extract_templates_and_params -# This is more accurate than our current regex, but only works -# if the user has already installed the library. -use_mwparserfromhell = True - # Pickle protocol version to use for storing dumps. # This config variable is not used for loading dumps. # Version 2 is common to both Python 2 and 3, and should diff --git a/pywikibot/textlib.py b/pywikibot/textlib.py index 36a5229..a52c569 100644 --- a/pywikibot/textlib.py +++ b/pywikibot/textlib.py @@ -22,7 +22,6 @@ import re

import pywikibot -from pywikibot import config2 as config from pywikibot.exceptions import InvalidTitle, SiteDefinitionError from pywikibot.family import Family from pywikibot.tools import ( @@ -1615,9 +1614,8 @@ parameters, and if this results multiple parameters with the same name only the last value provided will be returned.

- This uses the package L{mwparserfromhell} (mwpfh) if it is installed - and enabled by config.mwparserfromhell. Otherwise it falls back on a - regex based implementation. + This uses the package L{mwparserfromhell} (mwpfh) if it is installed. + Otherwise it falls back on a regex based implementation.

There are minor differences between the two implementations.

@@ -1636,33 +1634,29 @@ @type text: str @param remove_disabled_parts: Remove disabled wikitext such as comments and pre. If None (default), this is enabled when mwparserfromhell - is not available or is disabled in the config, and disabled if - mwparserfromhell is present and enabled in the config. + is not available and disabled if mwparserfromhell is present. @type remove_disabled_parts: bool or None @param strip: if enabled, strip arguments and values of templates. If None (default), this is enabled when mwparserfromhell - is not available or is disabled in the config, and disabled if - mwparserfromhell is present and enabled in the config. + is not available and disabled if mwparserfromhell is present. @type strip: bool @return: list of template name and params @rtype: list of tuple """ - use_mwparserfromhell = (config.use_mwparserfromhell - and not isinstance(mwparserfromhell, Exception)) + use_regex = isinstance(mwparserfromhell, Exception)

if remove_disabled_parts is None: - remove_disabled_parts = not use_mwparserfromhell - - if strip is None: - strip = not use_mwparserfromhell - + remove_disabled_parts = use_regex if remove_disabled_parts: text = removeDisabledParts(text)

- if use_mwparserfromhell: - return extract_templates_and_params_mwpfh(text, strip) - else: + if strip is None: + strip = use_regex + + if use_regex: return extract_templates_and_params_regex(text, False, strip) + else: + return extract_templates_and_params_mwpfh(text, strip)

def extract_templates_and_params_mwpfh(text, strip=False): @@ -1673,8 +1667,7 @@

Use extract_templates_and_params, which will select this mwparserfromhell implementation if based on whether the - mwparserfromhell package is installed and enabled by - config.mwparserfromhell. + mwparserfromhell package is installed.

@param text: The wikitext from which templates are extracted @type text: str diff --git a/tests/textlib_tests.py b/tests/textlib_tests.py index 1cf7791..9fb9461 100644 --- a/tests/textlib_tests.py +++ b/tests/textlib_tests.py @@ -15,11 +15,11 @@

import pywikibot import pywikibot.textlib as textlib -from pywikibot.textlib import _MultiTemplateMatchBuilder, extract_sections

-from pywikibot import config, UnknownSite from pywikibot.site import _IWEntry +from pywikibot.textlib import _MultiTemplateMatchBuilder, extract_sections from pywikibot.tools import suppress_warnings +from pywikibot import UnknownSite

from tests.aspects import ( unittest, require_modules, TestCase, DefaultDrySiteTestCase, @@ -725,7 +725,7 @@

def test_removing_disabled_parts_regex(self): """Test removing disabled parts when using the regex variant.""" - self.patch(config, 'use_mwparserfromhell', False) + self.patch(textlib, 'mwparserfromhell', Exception()) textlib.extract_templates_and_params('{{a<!-- -->}}', True) self.assertEqual(self._text, '{{a}}') self.assertFalse(self._mwpfh) @@ -739,7 +739,6 @@ @require_modules('mwparserfromhell') def test_removing_disabled_parts_mwpfh(self): """Test removing disabled parts when using the mwpfh variant.""" - self.patch(config, 'use_mwparserfromhell', True) textlib.extract_templates_and_params('{{a<!-- -->}}', True) self.assertEqual(self._text, '{{a}}') self.assertTrue(self._mwpfh) @@ -752,7 +751,7 @@

def test_strip_regex(self): """Test stripping values when using the regex variant.""" - self.patch(config, 'use_mwparserfromhell', False) + self.patch(textlib, 'mwparserfromhell', Exception()) textlib.extract_templates_and_params('{{a| foo }}', False, True) self.assertEqual(self._args, (False, True)) self.assertFalse(self._mwpfh) @@ -766,7 +765,6 @@ @require_modules('mwparserfromhell') def test_strip_mwpfh(self): """Test stripping values when using the mwpfh variant.""" - self.patch(config, 'use_mwparserfromhell', True) textlib.extract_templates_and_params('{{a| foo }}', None, True) self.assertEqual(self._args, (True, )) self.assertTrue(self._mwpfh)