jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/455396 )
Change subject: [bugfix] Fix the extended user-config extraction regex ......................................................................
[bugfix] Fix the extended user-config extraction regex
detached from I68a04e9198dc179f06c6245f1c1aec518603bd34
The current regex is greedy and finds one section only which accidentally holds all sections which is really unwanted and unnecessary and could lead to errors for not imported libraries.
generate_user_files.py: - DISABLED_SECTIONS are still excluded yet because there have malformed parts which is a script part but not a setting part and uses external libraries which shouldn't be used in the user-config.py. - OBSOLETE_SECTIONS are really obsolete either they are named as obsolete or the settings are already made by this script. - parse sections of config2.py by a new function and return a list of namedtuples containing head, info and content of a section. The info message (not used yet) is extracted from the first few lines after the header line and delimited by an empty line or an empty comment.
config2.py: - fix End of configuration section
generate_user_files_tests.py - tests added
Bug: T145371 Change-Id: I69a02be8723acdb8f08028760edbe8913c459929 --- M generate_user_files.py M pywikibot/config2.py M tests/generate_user_files_tests.py 3 files changed, 55 insertions(+), 33 deletions(-)
Approvals: Dalba: Looks good to me, approved jenkins-bot: Verified
diff --git a/generate_user_files.py b/generate_user_files.py index bef3b80..3033a18 100755 --- a/generate_user_files.py +++ b/generate_user_files.py @@ -18,6 +18,14 @@
from generate_family_file import _import_with_no_user_config
+# DISABLED_SECTIONS cannot be copied; variables must be set manually +DISABLED_SECTIONS = {'USER INTERFACE SETTINGS', # uses sys + 'EXTERNAL EDITOR SETTINGS', # uses os + } +OBSOLETE_SECTIONS = {'ACCOUNT SETTINGS', # already set + 'OBSOLETE SETTINGS', # obsolete + } + # Disable user-config usage as we are creating it here pywikibot = _import_with_no_user_config('pywikibot') config, __url__ = pywikibot.config2, pywikibot.__url__ @@ -230,40 +238,51 @@ {botpasswords}"""
-def copy_sections(): - """Take config sections and copying them to user-config.py. +def parse_sections(): + """Parse sections from config2.py file.
config2.py will be in the pywikibot/ directory relative to this generate_user_files script.
- @return: config text of all sections. - @rtype: str + @return: a list of ConfigSection named tuples. + @rtype: list """ + data = [] + ConfigSection = namedtuple('ConfigSection', 'head, info, section') + install = os.path.dirname(os.path.abspath(__file__)) with codecs.open(os.path.join(install, 'pywikibot', 'config2.py'), 'r', 'utf-8') as config_f: config_file = config_f.read()
result = re.findall( - '^(# ############# (?:' - 'LOGFILE|' - 'EXTERNAL SCRIPT PATH|' - 'INTERWIKI|' - 'SOLVE_DISAMBIGUATION|' - 'IMAGE RELATED|' - 'TABLE CONVERSION BOT|' - 'WEBLINK CHECKER|' - 'DATABASE|' - 'SEARCH ENGINE|' - 'COPYRIGHT|' - 'FURTHER' - ') SETTINGS .*)^(?=#####|# =====)', + '^(?P<section># #{5,} (?P<head>[A-Z][A-Z_ ]+[A-Z]) #{5,}\r?\n' + '(?:^#?\r?\n)?' # There may be an empty or short line after header + '(?P<comment>(?:^# .+?)+)' # first comment is used as help string + '^.*?)' # catch the remaining text + '^(?=# #{5,}|# ={5,})', # until section end marker config_file, re.MULTILINE | re.DOTALL)
- if not result: # Something is wrong with the regex - return None + for section, head, comment in result: + info = ' '.join(text.strip('# ') for text in comment.splitlines()) + data.append(ConfigSection(head, info, section)) + return data
- return '\n'.join(result) + +def copy_sections(): + """Take config sections and copy them to user-config.py. + + @return: config text of all selected sections. + @rtype: str + """ + result = [] + sections = parse_sections() + # copy settings + for section in filter(lambda x: x.head not in (DISABLED_SECTIONS + | OBSOLETE_SECTIONS), + sections): + result.append(section.section) + return ''.join(result)
def create_user_config(main_family, main_code, main_username, force=False): diff --git a/pywikibot/config2.py b/pywikibot/config2.py index 552f060..957ac47 100644 --- a/pywikibot/config2.py +++ b/pywikibot/config2.py @@ -894,6 +894,7 @@ # Version 4 is only available for Python 3.4 pickle_protocol = 2
+# ============================ # End of configuration section # ============================
diff --git a/tests/generate_user_files_tests.py b/tests/generate_user_files_tests.py index bde7497..f8c4750 100644 --- a/tests/generate_user_files_tests.py +++ b/tests/generate_user_files_tests.py @@ -10,7 +10,6 @@ import re
from tests.aspects import unittest, TestCase -from unittest import expectedFailure
import generate_user_files as guf
@@ -68,22 +67,23 @@ self.assertEqual(code, 'test') self.assertEqual(user, 'bar')
- @expectedFailure # T145371 - def test_copy_sections_fail(self): - """Test copy_sections function for sections not in config text.""" - config_text = guf.copy_sections() - for section in ('HTTP SETTINGS', - 'REPLICATION BOT SETTINGS', - ): - self.assertNotIn(section, config_text) + def test_parse_sections(self): + """Test parse_sections regex.""" + sections = guf.parse_sections() + self.assertGreater(len(sections), 10) + first = sections[0] + last = sections[-1] + self.assertEqual('ACCOUNT SETTINGS', first.head) + self.assertIn(first.head, first.section) + self.assertIn(first.info[:10], first.section) + self.assertEqual('OBSOLETE SETTINGS', last.head) + self.assertIn(last.head, last.section) + self.assertIn(last.info[:10], last.section)
def test_copy_sections_not_found(self): """Test copy_sections function for sections not in config text.""" config_text = guf.copy_sections() - for section in ('ACCOUNT SETTINGS', - 'OBSOLETE SETTINGS', - 'EXTERNAL EDITOR SETTINGS', - ): + for section in guf.DISABLED_SECTIONS | guf.OBSOLETE_SECTIONS: self.assertNotIn(section, config_text)
def test_copy_sections_found(self): @@ -94,6 +94,8 @@ 'EXTERNAL SCRIPT PATH SETTINGS', 'INTERWIKI SETTINGS', 'FURTHER SETTINGS', + 'HTTP SETTINGS', + 'REPLICATION BOT SETTINGS', ): self.assertIn(section, config_text) lines = config_text.splitlines()