Pywikibot-commits June 2021

pywikibot-commits@lists.wikimedia.org

1 participants
104 discussions

[Gerrit] ...core[master]: [IMPR] Double the wait time if ratelimit exceeded

by jenkins-bot (Code Review)

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/697589 ) Change subject: [IMPR] Double the wait time if ratelimit exceeded ...................................................................... [IMPR] Double the wait time if ratelimit exceeded Bug: T270912 Change-Id: I85bab755f0caa713bce26acb40c8106489bc6f50 --- M pywikibot/data/api.py 1 file changed, 8 insertions(+), 4 deletions(-) Approvals: Matěj Suchánek: Looks good to me, but someone else must approve Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index 932e5ae..48d376d 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -981,6 +981,7 @@ self.max_retries = pywikibot.config.max_retries else: self.max_retries = max_retries + self.current_retries = 0 if retry_wait is None: self.retry_wait = pywikibot.config.retry_wait else: @@ -1871,15 +1872,18 @@ def wait(self, delay=None): """Determine how long to wait after a failed request.""" - self.max_retries -= 1 - if self.max_retries < 0: + self.current_retries += 1 + if self.current_retries > self.max_retries: raise TimeoutError('Maximum retries attempted without success.') + + # double the next wait, but do not exceed config.retry_max seconds delay = delay or self.retry_wait + delay *= 2 ** (self.current_retries - 1) + delay = min(delay, config.retry_max) + pywikibot.warning('Waiting {:.1f} seconds before retrying.' .format(delay)) pywikibot.sleep(delay) - # double the next wait, but do not exceed config.retry_max seconds - self.retry_wait = min(config.retry_max, self.retry_wait * 2) class CachedRequest(Request): -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/697589 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I85bab755f0caa713bce26acb40c8106489bc6f50 Gerrit-Change-Number: 697589 Gerrit-PatchSet: 1 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: JJMC89 <JJMC89.Wikimedia(a)gmail.com> Gerrit-Reviewer: Matěj Suchánek <matejsuchanek97(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

2 years, 10 months

[Gerrit] ...core[master]: [tests] Do not call pywikibot.Site() if aspects.DisableSiteMixin is used

by jenkins-bot (Code Review)

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/697437 ) Change subject: [tests] Do not call pywikibot.Site() if aspects.DisableSiteMixin is used ...................................................................... [tests] Do not call pywikibot.Site() if aspects.DisableSiteMixin is used Bug: T284018 Change-Id: I291100ec2f6a552e0f1c4f793e28dd8290bf1d97 --- M pywikibot/bot.py M tests/aspects.py 2 files changed, 8 insertions(+), 4 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/bot.py b/pywikibot/bot.py index 216b39d..47422ae 100644 --- a/pywikibot/bot.py +++ b/pywikibot/bot.py @@ -349,10 +349,13 @@ # if user has enabled file logging, configure file handler if module_name in config.log or '*' in config.log: - # get PID - throttle = pywikibot.Site().throttle # initialize a Throttle object - pid = throttle.get_pid(module_name) # get the global PID if needed - pid = str(pid) + '-' if pid > 1 else '' + if pywikibot.Site.__doc__ == 'TEST': # set by aspects.DisableSiteMixin + pid = '' + else: + # get PID + throttle = pywikibot.Site().throttle # initialize a Throttle obj + pid = throttle.get_pid(module_name) # get the global PID if needed + pid = str(pid) + '-' if pid > 1 else '' if config.logfilename: # keep config.logfilename unchanged diff --git a/tests/aspects.py b/tests/aspects.py index f9da3cc..4c819fb 100644 --- a/tests/aspects.py +++ b/tests/aspects.py @@ -327,6 +327,7 @@ self.old_Site_lookup_method = pywikibot.Site pywikibot.Site = lambda *args: self.fail( '{}: Site() not permitted'.format(self.__class__.__name__)) + pywikibot.Site.__doc__ = 'TEST' super().setUp() -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/697437 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I291100ec2f6a552e0f1c4f793e28dd8290bf1d97 Gerrit-Change-Number: 697437 Gerrit-PatchSet: 1 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

2 years, 10 months

[Gerrit] ...core[master]: [doc] convert epytext docstrings to sphinx format

by jenkins-bot (Code Review)

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/673836 ) Change subject: [doc] convert epytext docstrings to sphinx format ...................................................................... [doc] convert epytext docstrings to sphinx format The conversion was made by a modified pywikibot_epytext_to_sphinx function docs/conf.py Bug: T278046 Change-Id: Ied73d3eb2367ec59747f0bc890d1c6640f4b278e --- M generate_family_file.py M generate_user_files.py M pwb.py M pywikibot/__init__.py M pywikibot/bot.py M pywikibot/bot_choice.py M pywikibot/comms/eventstreams.py M pywikibot/comms/http.py M pywikibot/config.py M pywikibot/cosmetic_changes.py M pywikibot/daemonize.py M pywikibot/data/api.py M pywikibot/data/mysql.py M pywikibot/data/sparql.py M pywikibot/data/wikistats.py M pywikibot/date.py M pywikibot/diff.py M pywikibot/echo.py M pywikibot/editor.py M pywikibot/exceptions.py M pywikibot/families/wikiquote_family.py M pywikibot/family.py M pywikibot/flow.py M pywikibot/i18n.py M pywikibot/interwiki_graph.py M pywikibot/logentries.py M pywikibot/logging.py M pywikibot/login.py M pywikibot/page/__init__.py M pywikibot/page/_collections.py M pywikibot/page/_revision.py M pywikibot/pagegenerators.py M pywikibot/proofreadpage.py M pywikibot/site/_apisite.py M pywikibot/site/_basesite.py M pywikibot/site/_datasite.py M pywikibot/site/_decorators.py M pywikibot/site/_extensions.py M pywikibot/site/_generators.py M pywikibot/site/_interwikimap.py M pywikibot/site/_namespace.py M pywikibot/site/_siteinfo.py M pywikibot/site/_tokenwallet.py M pywikibot/site_detect.py M pywikibot/specialbots/_upload.py M pywikibot/textlib.py M pywikibot/throttle.py M pywikibot/tools/__init__.py M pywikibot/tools/chars.py M pywikibot/tools/djvu.py M pywikibot/tools/formatter.py M pywikibot/userinterfaces/gui.py M pywikibot/userinterfaces/terminal_interface_base.py M pywikibot/userinterfaces/transliteration.py M pywikibot/userinterfaces/win32_unicode.py M pywikibot/version.py M pywikibot/xmlreader.py M scripts/add_text.py M scripts/archive/blockpageschecker.py M scripts/archive/capitalize_redirects.py M scripts/archive/catall.py M scripts/archive/commons_link.py M scripts/archive/create_categories.py M scripts/archive/data_ingestion.py M scripts/archive/disambredir.py M scripts/archive/editarticle.py M scripts/archive/flickrripper.py M scripts/archive/followlive.py M scripts/archive/freebasemappingupload.py M scripts/archive/imagecopy.py M scripts/archive/imagecopy_self.py M scripts/archive/imagerecat.py M scripts/archive/imageuncat.py M scripts/archive/isbn.py M scripts/archive/lonelypages.py M scripts/archive/makecat.py M scripts/archive/ndashredir.py M scripts/archive/piper.py M scripts/archive/selflink.py M scripts/archive/spamremove.py M scripts/archive/states_redirect.py M scripts/archive/surnames_redirects.py M scripts/archive/table2wiki.py M scripts/archive/unlink.py M scripts/archive/wikisourcetext.py M scripts/archivebot.py M scripts/basic.py M scripts/category.py M scripts/category_redirect.py M scripts/change_pagelang.py M scripts/checkimages.py M scripts/claimit.py M scripts/clean_sandbox.py M scripts/commons_information.py M scripts/commonscat.py M scripts/coordinate_import.py M scripts/cosmetic_changes.py M scripts/delete.py M scripts/djvutext.py M scripts/download_dump.py M scripts/fixing_redirects.py M scripts/harvest_template.py M scripts/illustrate_wikidata.py M scripts/image.py M scripts/imagetransfer.py M scripts/interwiki.py M scripts/interwikidata.py M scripts/listpages.py M scripts/login.py M scripts/maintenance/cache.py M scripts/maintenance/make_i18n_dict.py M scripts/misspelling.py M scripts/movepages.py M scripts/newitem.py M scripts/noreferences.py M scripts/nowcommons.py M scripts/pagefromfile.py M scripts/patrol.py M scripts/protect.py M scripts/redirect.py M scripts/reflinks.py M scripts/replace.py M scripts/replicate_wiki.py M scripts/revertbot.py M scripts/solve_disambiguation.py M scripts/speedy_delete.py M scripts/template.py M scripts/templatecount.py M scripts/touch.py M scripts/transferbot.py M scripts/unusedfiles.py M scripts/upload.py M scripts/watchlist.py M scripts/weblinkchecker.py M scripts/welcome.py M setup.py M tests/aspects.py M tests/bot_tests.py M tests/deprecation_tests.py M tests/link_tests.py M tests/page_tests.py M tests/pagegenerators_tests.py M tests/replacebot_tests.py M tests/site_detect_tests.py M tests/textlib_tests.py M tests/tools_formatter_tests.py M tests/ui_tests.py M tests/uploadscript_tests.py M tests/utils.py M tests/wikibase_edit_tests.py 150 files changed, 3,888 insertions(+), 3,869 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/673836 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: Ied73d3eb2367ec59747f0bc890d1c6640f4b278e Gerrit-Change-Number: 673836 Gerrit-PatchSet: 17 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Damian <atagar1(a)gmail.com> Gerrit-Reviewer: Huji <huji.huji(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

2 years, 10 months

[Gerrit] ...core[master]: [cleanup] remove deprecated extract_templates_and_params_* functions

by jenkins-bot (Code Review)

jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/697143 ) Change subject: [cleanup] remove deprecated extract_templates_and_params_* functions ...................................................................... [cleanup] remove deprecated extract_templates_and_params_* functions mwparserfromhell or wikitextparser MediaWiki markup parser is mandatory. Therefore: - remove deprecated extract_templates_and_params_mwpfh function - remove deprecated extract_templates_and_params_regex function - remove tests accordingly Change-Id: I529e846e655cbceb21672b09b2329c11e25d3307 --- M pywikibot/textlib.py M tests/textlib_tests.py 2 files changed, 70 insertions(+), 324 deletions(-) Approvals: JJMC89: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/textlib.py b/pywikibot/textlib.py index 696c6c3..bfb1ee2 100644 --- a/pywikibot/textlib.py +++ b/pywikibot/textlib.py @@ -55,13 +55,6 @@ # cache for replaceExcept to avoid recompile or regexes each call _regex_cache = {} -# This regex is only for use by extract_templates_and_params_regex. -# It does not support template variables consisting of nested templates, -# system variables like {{CURRENTYEAR}}, or template variables like {{{1}}}. -_ETP_REGEX = re.compile( - r'{{(?:msg:)?(?P<name>[^{\|]+?)' - r'(?:\|(?P<params>[^{]+?(?:{[^{]+?}[^{]*?)?)?)?}}') - # The regex below collects nested templates, providing simpler # identification of templates used at the top-level of wikitext. # It doesn't match {{{1|...}}}, however it also does not match templates @@ -1662,177 +1655,6 @@ return result -@deprecated('extract_templates_and_params', since='20210329', - future_warning=True) -def extract_templates_and_params_mwpfh(text: str, - strip: bool = False) -> ETPType: - """DEPRECATED. Extract templates with params using mwparserfromhell.""" - global wikitextparser - saved_parser = wikitextparser - import mwparserfromhell as wikitextparser - result = extract_templates_and_params(text, strip=strip) - wikitextparser = saved_parser - return result - - -@deprecated('extract_templates_and_params', since='20210331', - future_warning=True) -def extract_templates_and_params_regex(text: str, - remove_disabled_parts: bool = True, - strip: bool = True) -> ETPType: - """DEPRECATED. Extract templates with params using a regex. - - This function should not be called directly. - - Use extract_templates_and_params, which will fallback to using this - regex based implementation when the mwparserfromhell implementation - is not used. - - @param text: The wikitext from which templates are extracted - @param strip: if enabled, strip arguments and values of templates - @return: list of template name and params - """ - # remove commented-out stuff etc. - if remove_disabled_parts: - thistxt = removeDisabledParts(text) - else: - thistxt = text - - # marker for inside templates or parameters - marker1 = findmarker(thistxt) - - # marker for links - marker2 = findmarker(thistxt, '##', '#') - - # marker for math - marker3 = findmarker(thistxt, '%%', '%') - - # marker for value parameter - marker4 = findmarker(thistxt, '§§', '§') - - result = [] - Rmath = re.compile(r'<math>[^<]+</math>') - Rvalue = re.compile(r'{{{.+?}}}') - Rmarker1 = re.compile(r'{m}(\d+){m}'.format(m=marker1)) - Rmarker2 = re.compile(r'{m}(\d+){m}'.format(m=marker2)) - Rmarker3 = re.compile(r'{m}(\d+){m}'.format(m=marker3)) - Rmarker4 = re.compile(r'{m}(\d+){m}'.format(m=marker4)) - - # Replace math with markers - maths = {} - count = 0 - for m in Rmath.finditer(thistxt): - count += 1 - item = m.group() - thistxt = thistxt.replace(item, '{m}{c}{m}' - .format(m=marker3, c=count)) - maths[count] = item - - values = {} - count = 0 - for m in Rvalue.finditer(thistxt): - count += 1 - # If we have digits between brackets, restoring from dict may fail. - # So we need to change the index. We have to search in the origin text. - while '}}}%d{{{' % count in text: - count += 1 - item = m.group() - thistxt = thistxt.replace(item, '{m}{c}{m}' - .format(m=marker4, c=count)) - values[count] = item - - inside = {} - seen = set() - count = 0 - while _ETP_REGEX.search(thistxt) is not None: - for m in _ETP_REGEX.finditer(thistxt): - # Make sure it is not detected again - item = m.group() - if item in seen: - continue # speed up - seen.add(item) - count += 1 - while '}}%d{{' % count in text: - count += 1 - thistxt = thistxt.replace(item, '{m}{c}{m}' - .format(m=marker1, c=count)) - - # Make sure stored templates don't contain markers - for m2 in Rmarker1.finditer(item): - item = item.replace(m2.group(), inside[int(m2.group(1))]) - for m2 in Rmarker3.finditer(item): - item = item.replace(m2.group(), maths[int(m2.group(1))]) - for m2 in Rmarker4.finditer(item): - item = item.replace(m2.group(), values[int(m2.group(1))]) - inside[count] = item - - # Name - name = m.group('name').strip() - m2 = Rmarker1.search(name) or Rmath.search(name) - if m2 is not None: - # Doesn't detect templates whose name changes, - # or templates whose name contains math tags - continue - - # {{#if: }} - if not name or name.startswith('#'): - continue - - # Parameters - paramString = m.group('params') - params = OrderedDict() - numbered_param = 1 - if paramString: - # Replace wikilinks with markers - links = {} - count2 = 0 - for m2 in pywikibot.link_regex.finditer(paramString): - count2 += 1 - item = m2.group(0) - paramString = paramString.replace( - item, '{m}{c}{m}'.format(m=marker2, c=count2)) - links[count2] = item - # Parse string - markedParams = paramString.split('|') - # Replace markers - for param in markedParams: - if '=' in param: - param_name, param_val = param.split('=', 1) - implicit_parameter = False - else: - param_name = str(numbered_param) - param_val = param - numbered_param += 1 - implicit_parameter = True - count = len(inside) - for m2 in Rmarker1.finditer(param_val): - param_val = param_val.replace(m2.group(), - inside[int(m2.group(1))]) - for m2 in Rmarker2.finditer(param_val): - param_val = param_val.replace(m2.group(), - links[int(m2.group(1))]) - for m2 in Rmarker3.finditer(param_val): - param_val = param_val.replace(m2.group(), - maths[int(m2.group(1))]) - for m2 in Rmarker4.finditer(param_val): - param_val = param_val.replace(m2.group(), - values[int(m2.group(1))]) - if strip: - param_name = param_name.strip() - if not implicit_parameter: - param_val = param_val.strip() - params[param_name] = param_val - - # Special case for {{a|}} which has an undetected parameter - if not params and '|' in m.group(0): - params = OrderedDict({'1': ''}) - - # Add it to the result - result.append((name, params)) - - return result - - def extract_templates_and_params_regex_simple(text: str): """ Extract top-level templates with params using only a simple regex. diff --git a/tests/textlib_tests.py b/tests/textlib_tests.py index 6904e2c..e53c245 100644 --- a/tests/textlib_tests.py +++ b/tests/textlib_tests.py @@ -18,7 +18,7 @@ from pywikibot.exceptions import UnknownSiteError from pywikibot.site._interwikimap import _IWEntry from pywikibot.textlib import MultiTemplateMatchBuilder, extract_sections -from pywikibot.tools import suppress_warnings +from pywikibot.tools import has_module, suppress_warnings from tests import mock from tests.aspects import ( DefaultDrySiteTestCase, @@ -303,10 +303,6 @@ 'Invalid category title extracted: nasty{{{!}}') -WARNING_MSG = (r'.*extract_templates_and_params_.*' - r'is deprecated for .*; use extract_templates_and_params') - - class TestTemplateParams(TestCase): """Test to verify that template params extraction works.""" @@ -315,61 +311,58 @@ def _common_results(self, func): """Common cases.""" - with suppress_warnings(WARNING_MSG, category=FutureWarning): - self.assertEqual(func('{{a}}'), [('a', OrderedDict())]) + self.assertEqual(func('{{a}}'), [('a', OrderedDict())]) + self.assertEqual(func('{{ a}}'), [('a', OrderedDict())]) + self.assertEqual(func('{{a }}'), [('a', OrderedDict())]) + self.assertEqual(func('{{ a }}'), [('a', OrderedDict())]) + self.assertEqual(func('{{a|b=c}}'), + [('a', OrderedDict((('b', 'c'), )))]) + self.assertEqual(func('{{a|b|c=d}}'), + [('a', OrderedDict((('1', 'b'), ('c', 'd'))))]) + self.assertEqual(func('{{a|b=c|f=g|d=e|1=}}'), + [('a', OrderedDict((('b', 'c'), ('f', 'g'), + ('d', 'e'), ('1', ''))))]) + self.assertEqual(func('{{a|1=2|c=d}}'), + [('a', OrderedDict((('1', '2'), ('c', 'd'))))]) + self.assertEqual(func('{{a|c=d|1=2}}'), + [('a', OrderedDict((('c', 'd'), ('1', '2'))))]) + self.assertEqual(func('{{a|5=d|a=b}}'), + [('a', OrderedDict((('5', 'd'), ('a', 'b'))))]) + self.assertEqual(func('{{a|=2}}'), + [('a', OrderedDict((('', '2'), )))]) + self.assertEqual(func('{{a|}}'), + [('a', OrderedDict((('1', ''), )))]) + self.assertEqual(func('{{a|=|}}'), + [('a', OrderedDict((('', ''), ('1', ''))))]) + self.assertEqual(func('{{a||}}'), + [('a', OrderedDict((('1', ''), ('2', ''))))]) + self.assertEqual(func('{{a|b={{{1}}}}}'), + [('a', OrderedDict((('b', '{{{1}}}'), )))]) + self.assertEqual(func('{{a|b=<noinclude>{{{1}}}</noinclude>}}'), + [('a', + OrderedDict((('b', + '<noinclude>{{{1}}}</noinclude>'), + )))]) + self.assertEqual(func('{{Template:a|b=c}}'), + [('Template:a', OrderedDict((('b', 'c'), )))]) + self.assertEqual(func('{{template:a|b=c}}'), + [('template:a', OrderedDict((('b', 'c'), )))]) + self.assertEqual(func('{{:a|b=c}}'), + [(':a', OrderedDict((('b', 'c'), )))]) + self.assertEqual(func('{{a|b={{{1}}}|c={{{2}}}}}'), + [('a', OrderedDict((('b', '{{{1}}}'), + ('c', '{{{2}}}'))))]) + self.assertEqual(func('{{a|b=c}}{{d|e=f}}'), + [('a', OrderedDict((('b', 'c'), ))), + ('d', OrderedDict((('e', 'f'), )))]) - with suppress_warnings(WARNING_MSG, category=FutureWarning): - self.assertEqual(func('{{ a}}'), [('a', OrderedDict())]) - self.assertEqual(func('{{a }}'), [('a', OrderedDict())]) - self.assertEqual(func('{{ a }}'), [('a', OrderedDict())]) - self.assertEqual(func('{{a|b=c}}'), - [('a', OrderedDict((('b', 'c'), )))]) - self.assertEqual(func('{{a|b|c=d}}'), - [('a', OrderedDict((('1', 'b'), ('c', 'd'))))]) - self.assertEqual(func('{{a|b=c|f=g|d=e|1=}}'), - [('a', OrderedDict((('b', 'c'), ('f', 'g'), - ('d', 'e'), ('1', ''))))]) - self.assertEqual(func('{{a|1=2|c=d}}'), - [('a', OrderedDict((('1', '2'), ('c', 'd'))))]) - self.assertEqual(func('{{a|c=d|1=2}}'), - [('a', OrderedDict((('c', 'd'), ('1', '2'))))]) - self.assertEqual(func('{{a|5=d|a=b}}'), - [('a', OrderedDict((('5', 'd'), ('a', 'b'))))]) - self.assertEqual(func('{{a|=2}}'), - [('a', OrderedDict((('', '2'), )))]) - self.assertEqual(func('{{a|}}'), - [('a', OrderedDict((('1', ''), )))]) - self.assertEqual(func('{{a|=|}}'), - [('a', OrderedDict((('', ''), ('1', ''))))]) - self.assertEqual(func('{{a||}}'), - [('a', OrderedDict((('1', ''), ('2', ''))))]) - self.assertEqual(func('{{a|b={{{1}}}}}'), - [('a', OrderedDict((('b', '{{{1}}}'), )))]) - self.assertEqual(func('{{a|b=<noinclude>{{{1}}}</noinclude>}}'), - [('a', - OrderedDict((('b', - '<noinclude>{{{1}}}</noinclude>'), - )))]) - self.assertEqual(func('{{Template:a|b=c}}'), - [('Template:a', OrderedDict((('b', 'c'), )))]) - self.assertEqual(func('{{template:a|b=c}}'), - [('template:a', OrderedDict((('b', 'c'), )))]) - self.assertEqual(func('{{:a|b=c}}'), - [(':a', OrderedDict((('b', 'c'), )))]) - self.assertEqual(func('{{a|b={{{1}}}|c={{{2}}}}}'), - [('a', OrderedDict((('b', '{{{1}}}'), - ('c', '{{{2}}}'))))]) - self.assertEqual(func('{{a|b=c}}{{d|e=f}}'), - [('a', OrderedDict((('b', 'c'), ))), - ('d', OrderedDict((('e', 'f'), )))]) + # initial '{' and '}' should be ignored as outer wikitext + self.assertEqual(func('{{{a|b}}X}'), + [('a', OrderedDict((('1', 'b'), )))]) - # initial '{' and '}' should be ignored as outer wikitext - self.assertEqual(func('{{{a|b}}X}'), - [('a', OrderedDict((('1', 'b'), )))]) - - # sf.net bug 1575: unclosed template - self.assertEqual(func('{{a'), []) - self.assertEqual(func('{{a}}{{foo|'), [('a', OrderedDict())]) + # sf.net bug 1575: unclosed template + self.assertEqual(func('{{a'), []) + self.assertEqual(func('{{a}}{{foo|'), [('a', OrderedDict())]) def _unstripped(self, func): """Common cases of unstripped results.""" @@ -456,11 +449,12 @@ ('2', 'd')])), ('b', OrderedDict([('1', 'c')]))]) - def _mwpfh_passes(self, func, failing=False): + def _mwpfh_passes(self, func): """Common cases failing with wikitextparser but passes with mwpfh. Probably the behaviour of regex or mwpfh is wrong. """ + failing = has_module('wikitextparser') patterns = [ '{{subst:a|b=c}}', '{{safesubst:a|b=c}}', @@ -480,25 +474,24 @@ @require_modules('mwparserfromhell') def test_extract_templates_params_mwpfh(self): """Test using mwparserfromhell.""" - func = textlib.extract_templates_and_params_mwpfh - with suppress_warnings(WARNING_MSG, category=FutureWarning): - self._common_results(func) - self._order_differs(func) - self._unstripped(func) - self._etp_regex_differs(func) - self._mwpfh_passes(func) + func = textlib.extract_templates_and_params + self._common_results(func) + self._order_differs(func) + self._unstripped(func) + self._etp_regex_differs(func) + self._mwpfh_passes(func) - self.assertCountEqual(func('{{a|{{c|{{d}}}}}}'), - [('c', OrderedDict((('1', '{{d}}'), ))), - ('a', OrderedDict([('1', '{{c|{{d}}}}')])), - ('d', OrderedDict()) - ]) + self.assertCountEqual(func('{{a|{{c|{{d}}}}}}'), + [('c', OrderedDict((('1', '{{d}}'), ))), + ('a', OrderedDict([('1', '{{c|{{d}}}}')])), + ('d', OrderedDict()) + ]) - self.assertCountEqual(func('{{a|{{c|{{d|}}}}}}'), - [('c', OrderedDict((('1', '{{d|}}'), ))), - ('a', OrderedDict([('1', '{{c|{{d|}}}}')])), - ('d', OrderedDict([('1', '')])) - ]) + self.assertCountEqual(func('{{a|{{c|{{d|}}}}}}'), + [('c', OrderedDict((('1', '{{d|}}'), ))), + ('a', OrderedDict([('1', '{{c|{{d|}}}}')])), + ('d', OrderedDict([('1', '')])) + ]) @require_modules('mwparserfromhell') def test_extract_templates_params_parser_stripped(self): @@ -518,7 +511,7 @@ self._order_differs(func) self._unstripped(func) self._etp_regex_differs(func) - self._mwpfh_passes(func, failing=True) + self._mwpfh_passes(func) self.assertCountEqual(func('{{a|{{c|{{d}}}}}}'), [('c', OrderedDict((('1', '{{d}}'), ))), @@ -532,42 +525,6 @@ ('d', OrderedDict([('1', '')])) ]) - def test_extract_templates_params_regex(self): - """Test using many complex regexes.""" - func = functools.partial(textlib.extract_templates_and_params_regex, - remove_disabled_parts=False, strip=False) - with suppress_warnings(WARNING_MSG, category=FutureWarning): - self._common_results(func) - self._order_differs(func) - self._unstripped(func) - # FIXME: {} is normal text - self.assertEqual(func('{{a|b={} }}'), []) - - def test_extract_templates_params_regex_stripped(self): - """Test using many complex regexes with stripping.""" - func = textlib.extract_templates_and_params_regex - with suppress_warnings(WARNING_MSG, category=FutureWarning): - self._common_results(func) - self._order_differs(func) - self._stripped(func) - - self.assertEqual(func('{{a|b=}}'), - [('a', OrderedDict((('b', ''), )))]) - - # Identical to mwpfh - self.assertCountEqual(func('{{a|{{c|{{d}}}}}}'), - [('c', OrderedDict((('1', '{{d}}'), ))), - ('a', OrderedDict([('1', '{{c|{{d}}}}')])), - ('d', OrderedDict()) - ]) - - # However fails to correctly handle three levels of balanced - # brackets with empty parameters - self.assertCountEqual(func('{{a|{{c|{{d|}}}}}}'), - [('c', OrderedDict((('1', '{{d|}}}'), ))), - ('d', OrderedDict([('1', '}')])) - ]) - @require_modules('mwparserfromhell') def test_extract_templates_params(self): """Test that the normal entry point works.""" @@ -621,39 +578,6 @@ self.assertEqual(func('{{a|{{c|{{d|{{e|}}}} }} }} foo {{b}}'), [(None, OrderedDict())]) - def test_etp_regex(self): - """Test _ETP_REGEX.""" - func = textlib._ETP_REGEX.search - - self.assertIsNotNone(func('{{{1}}}')) - self.assertIsNotNone(func('{{a|b={{{1}}} }}')) - self.assertIsNotNone(func('{{a|b={{c}} }}')) - self.assertIsNotNone(func('{{a|b={{c}} }}')) - self.assertIsNotNone(func('{{a|b={{c|d=1}} }}')) - - self.assertIsNotNone(func('{{a|{{c}} }}')) - self.assertIsNotNone(func('{{a|{{c|d}} }}')) - - func = textlib._ETP_REGEX.match - - self.assertIsNone(func('{{{1}}}')) - - self.assertIsNotNone(func('{{#if:foo}}')) - self.assertIsNotNone(func('{{foo:}}')) - - self.assertIsNotNone(func('{{CURRENTYEAR}}')) - self.assertIsNotNone(func('{{1}}')) - - self.assertIsNone(func('{{a|b={{CURRENTYEAR}} }}')) - self.assertIsNone(func('{{a|b={{{1}}} }}')) - self.assertIsNone(func('{{a|b={{c}} }}')) - self.assertIsNone(func('{{a|b={{c|d=1}} }}')) - self.assertIsNone(func('{{a|b={} }}')) - self.assertIsNone(func('{{:a|b={{c|d=1}} }}')) - - self.assertIsNone(func('{{a|{{c}} }}')) - self.assertIsNone(func('{{a|{{c|d}} }}')) - def test_nested_template_regex_search(self): """Test NESTED_TEMPLATE_REGEX search.""" func = textlib.NESTED_TEMPLATE_REGEX.search -- To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/697143 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Change-Id: I529e846e655cbceb21672b09b2329c11e25d3307 Gerrit-Change-Number: 697143 Gerrit-PatchSet: 3 Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: JJMC89 <JJMC89.Wikimedia(a)gmail.com> Gerrit-Reviewer: jenkins-bot Gerrit-MessageType: merged

2 years, 10 months

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

Pywikibot-commits June 2021