jenkins-bot merged this change.
[bugfix] Fix the extended user-config extraction regex
detached from I68a04e9198dc179f06c6245f1c1aec518603bd34
The current regex is greedy and finds one section only which accidentally
holds all sections which is really unwanted and unnecessary and could
lead to errors for not imported libraries.
generate_user_files.py:
- DISABLED_SECTIONS are still excluded yet because there have malformed parts
which is a script part but not a setting part and uses external libraries
which shouldn't be used in the user-config.py.
- OBSOLETE_SECTIONS are really obsolete either they are named as obsolete
or the settings are already made by this script.
- parse sections of config2.py by a new function and return a list of
namedtuples containing head, info and content of a section. The info
message (not used yet) is extracted from the first few lines after the
header line and delimited by an empty line or an empty comment.
config2.py:
- fix End of configuration section
generate_user_files_tests.py
- tests added
Bug: T145371
Change-Id: I69a02be8723acdb8f08028760edbe8913c459929
---
M generate_user_files.py
M pywikibot/config2.py
M tests/generate_user_files_tests.py
3 files changed, 55 insertions(+), 33 deletions(-)
diff --git a/generate_user_files.py b/generate_user_files.py
index bef3b80..3033a18 100755
--- a/generate_user_files.py
+++ b/generate_user_files.py
@@ -18,6 +18,14 @@
from generate_family_file import _import_with_no_user_config
+# DISABLED_SECTIONS cannot be copied; variables must be set manually
+DISABLED_SECTIONS = {'USER INTERFACE SETTINGS', # uses sys
+ 'EXTERNAL EDITOR SETTINGS', # uses os
+ }
+OBSOLETE_SECTIONS = {'ACCOUNT SETTINGS', # already set
+ 'OBSOLETE SETTINGS', # obsolete
+ }
+
# Disable user-config usage as we are creating it here
pywikibot = _import_with_no_user_config('pywikibot')
config, __url__ = pywikibot.config2, pywikibot.__url__
@@ -230,40 +238,51 @@
{botpasswords}"""
-def copy_sections():
- """Take config sections and copying them to user-config.py.
+def parse_sections():
+ """Parse sections from config2.py file.
config2.py will be in the pywikibot/ directory relative to this
generate_user_files script.
- @return: config text of all sections.
- @rtype: str
+ @return: a list of ConfigSection named tuples.
+ @rtype: list
"""
+ data = []
+ ConfigSection = namedtuple('ConfigSection', 'head, info, section')
+
install = os.path.dirname(os.path.abspath(__file__))
with codecs.open(os.path.join(install, 'pywikibot', 'config2.py'),
'r', 'utf-8') as config_f:
config_file = config_f.read()
result = re.findall(
- '^(# ############# (?:'
- 'LOGFILE|'
- 'EXTERNAL SCRIPT PATH|'
- 'INTERWIKI|'
- 'SOLVE_DISAMBIGUATION|'
- 'IMAGE RELATED|'
- 'TABLE CONVERSION BOT|'
- 'WEBLINK CHECKER|'
- 'DATABASE|'
- 'SEARCH ENGINE|'
- 'COPYRIGHT|'
- 'FURTHER'
- ') SETTINGS .*)^(?=#####|# =====)',
+ '^(?P<section># #{5,} (?P<head>[A-Z][A-Z_ ]+[A-Z]) #{5,}\r?\n'
+ '(?:^#?\r?\n)?' # There may be an empty or short line after header
+ '(?P<comment>(?:^# .+?)+)' # first comment is used as help string
+ '^.*?)' # catch the remaining text
+ '^(?=# #{5,}|# ={5,})', # until section end marker
config_file, re.MULTILINE | re.DOTALL)
- if not result: # Something is wrong with the regex
- return None
+ for section, head, comment in result:
+ info = ' '.join(text.strip('# ') for text in comment.splitlines())
+ data.append(ConfigSection(head, info, section))
+ return data
- return '\n'.join(result)
+
+def copy_sections():
+ """Take config sections and copy them to user-config.py.
+
+ @return: config text of all selected sections.
+ @rtype: str
+ """
+ result = []
+ sections = parse_sections()
+ # copy settings
+ for section in filter(lambda x: x.head not in (DISABLED_SECTIONS
+ | OBSOLETE_SECTIONS),
+ sections):
+ result.append(section.section)
+ return ''.join(result)
def create_user_config(main_family, main_code, main_username, force=False):
diff --git a/pywikibot/config2.py b/pywikibot/config2.py
index 552f060..957ac47 100644
--- a/pywikibot/config2.py
+++ b/pywikibot/config2.py
@@ -894,6 +894,7 @@
# Version 4 is only available for Python 3.4
pickle_protocol = 2
+# ============================
# End of configuration section
# ============================
diff --git a/tests/generate_user_files_tests.py b/tests/generate_user_files_tests.py
index bde7497..f8c4750 100644
--- a/tests/generate_user_files_tests.py
+++ b/tests/generate_user_files_tests.py
@@ -10,7 +10,6 @@
import re
from tests.aspects import unittest, TestCase
-from unittest import expectedFailure
import generate_user_files as guf
@@ -68,22 +67,23 @@
self.assertEqual(code, 'test')
self.assertEqual(user, 'bar')
- @expectedFailure # T145371
- def test_copy_sections_fail(self):
- """Test copy_sections function for sections not in config text."""
- config_text = guf.copy_sections()
- for section in ('HTTP SETTINGS',
- 'REPLICATION BOT SETTINGS',
- ):
- self.assertNotIn(section, config_text)
+ def test_parse_sections(self):
+ """Test parse_sections regex."""
+ sections = guf.parse_sections()
+ self.assertGreater(len(sections), 10)
+ first = sections[0]
+ last = sections[-1]
+ self.assertEqual('ACCOUNT SETTINGS', first.head)
+ self.assertIn(first.head, first.section)
+ self.assertIn(first.info[:10], first.section)
+ self.assertEqual('OBSOLETE SETTINGS', last.head)
+ self.assertIn(last.head, last.section)
+ self.assertIn(last.info[:10], last.section)
def test_copy_sections_not_found(self):
"""Test copy_sections function for sections not in config text."""
config_text = guf.copy_sections()
- for section in ('ACCOUNT SETTINGS',
- 'OBSOLETE SETTINGS',
- 'EXTERNAL EDITOR SETTINGS',
- ):
+ for section in guf.DISABLED_SECTIONS | guf.OBSOLETE_SECTIONS:
self.assertNotIn(section, config_text)
def test_copy_sections_found(self):
@@ -94,6 +94,8 @@
'EXTERNAL SCRIPT PATH SETTINGS',
'INTERWIKI SETTINGS',
'FURTHER SETTINGS',
+ 'HTTP SETTINGS',
+ 'REPLICATION BOT SETTINGS',
):
self.assertIn(section, config_text)
lines = config_text.splitlines()
To view, visit change 455396. To unsubscribe, or for help writing mail filters, visit settings.