jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/908966 )
Change subject: [bugfix] Remove sites from FamilyFileGenerator.langs on error ......................................................................
[bugfix] Remove sites from FamilyFileGenerator.langs on error
- If loading a site fails, remove it from langs attribute of FamilyFileGenerator - [s]trict: add the ability to process sites with the same domain only - with [e]dit sites can be given delimited by space, comma or both - collect self.prefixes for tests after FamilyFileGenerator.getapis() was processed - use [s]trict option for Wikimedia site for within tests - remove old implementation removing obsolete sites in tests
Bug: T334775 Bug: T334714 Change-Id: I5b049ead55e19f2a4c2ba522f34412e541737669 --- M tests/generate_family_file_tests.py M pywikibot/scripts/generate_family_file.py 2 files changed, 79 insertions(+), 22 deletions(-)
Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified
diff --git a/pywikibot/scripts/generate_family_file.py b/pywikibot/scripts/generate_family_file.py index d5c00cc..8b0607c 100755 --- a/pywikibot/scripts/generate_family_file.py +++ b/pywikibot/scripts/generate_family_file.py @@ -12,7 +12,7 @@
<url>: an url from where the family settings are loaded <name>: the family name without "_family.py" tail. - <dointerwiki>: predefined answer (y|n) to add multiple site codes + <dointerwiki>: predefined answer (y|s|n) to add multiple site codes <verify>: disable certificate validaton `(y|n)
Example:: @@ -25,14 +25,18 @@ .. versionchanged:: 7.0 moved to pywikibot.scripts folder; create family files in families folder of your base directory instead of pywikibot/families. +.. versionchanged:: 8.1 + [s]trict can be given for <dointerwiki> parameter to ensure that + sites are from the given domain. """ # -# (C) Pywikibot team, 2010-2022 +# (C) Pywikibot team, 2010-2023 # # Distributed under the terms of the MIT license # import codecs import os +import re import string import sys from contextlib import suppress @@ -63,8 +67,10 @@ :param url: an url from where the family settings are loaded :param name: the family name without "_family.py" tail. :param dointerwiki: Predefined answer to add multiple site - codes. Pass `Y` or `y` for yes `N` or `n` for no and - `E` or `e` if you want to edit the collection of sites. + codes. Pass `Y` or `y` for yes, `S` or `s` for strict which + only includes site of the same domain (usually for Wikimedia + sites), `N` or `n` for no and `E` or `e` if you want to edit + the collection of sites. :param verify: If a certificate verification failes, you may pass `Y` or `y` to disable certificate validaton `N` or `n` to keep it enabled. @@ -149,7 +155,14 @@ self.writefile(verify)
def getlangs(self, w) -> None: - """Determine site code of a family.""" + """Determine site code of a family. + + .. versionchanged:: 8.1 + with [e]dit the interwiki list can be given delimited by + space or comma or both. With [s]trict only sites with the + same domain are collected. A [h]elp answer was added to show + more information about possible answers. + """ print('Determining other sites...', end='') try: self.langs = w.langs @@ -169,21 +182,39 @@ code_len = len(self.langs) if code_len > 1: if self.dointerwiki is None: - makeiw = input( - '\nThere are {} sites available.' - '\nDo you want to generate interwiki links? ' - 'This might take a long time. ([y]es/[N]o/[e]dit)' - .format(code_len)).lower() + while True: + makeiw = input( + '\n' + f'There are {code_len} sites available.' + ' Do you want to generate interwiki links?\n' + 'This might take a long time. ' + '([y]es, [s]trict, [N]o, [e]dit), [h]elp) ').lower() + if makeiw in ('y', 's', 'n', 'e', ''): + break + print( + '\n' + '[y]es: create interwiki links for all sites\n' + '[s]trict: yes, but for sites with same domain only\n' + '[N]o: no, use the current site only (default)\n' + '[e]dit: get a list delimited with space or comma\n' + '[h]elp: this help message' + ) else: makeiw = self.dointerwiki
- if makeiw == 'n': + if makeiw in ('n', ''): self.langs = [wiki for wiki in self.langs if wiki['url'] == w.iwpath] + elif makeiw == 's': + domain = '.'.join(urlparse(w.server).hostname.split('.')[1:]) + self.langs = [wiki for wiki in self.langs + if domain in wiki['url']] + elif makeiw == 'e': for wiki in self.langs: print(wiki['prefix'], wiki['url']) - do_langs = input('Which sites do you want: ') + do_langs = re.split(' *,| +', + input('Which sites do you want: ')) self.langs = [wiki for wiki in self.langs if wiki['prefix'] in do_langs or wiki['url'] == w.iwpath] @@ -196,7 +227,8 @@
def getapis(self) -> None: """Load other site pages.""" - print('Loading wikis... ') + print(f'Loading {len(self.langs)} wikis... ') + remove = [] for lang in self.langs: key = lang['prefix'] print(f' * {key}... ', end='') @@ -206,17 +238,21 @@ print('downloaded') except Exception as e: # pragma: no cover print(e) + remove.append(lang) else: print('in cache')
+ for lang in remove: + self.langs.remove(lang) + def writefile(self, verify) -> None: """Write the family file.""" fn = os.path.join(self.base_dir, 'families', f'{self.name}_family.py') print(f'Writing {fn}... ')
- if os.path.exists(fn) and input('{} already exists. Overwrite? (y/n)' - .format(fn)).lower() == 'n': + if os.path.exists(fn) and input( + f'{fn} already exists. Overwrite? (y/n) ').lower() == 'n': print('Terminating.') sys.exit(1)
diff --git a/tests/generate_family_file_tests.py b/tests/generate_family_file_tests.py index f219ac6..e97f884 100755 --- a/tests/generate_family_file_tests.py +++ b/tests/generate_family_file_tests.py @@ -11,6 +11,7 @@ from urllib.parse import urlparse
from pywikibot import Site +from pywikibot.family import WikimediaFamily from pywikibot.scripts import generate_family_file from tests.aspects import DefaultSiteTestCase from tests.utils import skipping @@ -29,15 +30,15 @@ self.langs.append(wiki) break
- self.prefixes = [item['prefix'] for item in self.langs] super().getapis() + self.prefixes = [item['prefix'] for item in self.langs] self.langs = save
def writefile(self, verify): """Pass writing."""
-class TestGenerateFamilyFiles(DefaultSiteTestCase): +class TestGenerateFamilyFile(DefaultSiteTestCase):
"""Test generate_family_file functionality."""
@@ -55,15 +56,17 @@ def setUp(self): """Set up tests.""" super().setUp() + answer = 's' if isinstance(self.site.family, WikimediaFamily) else 'y' self.generator_instance = FamilyTestGenerator( - url=self.site.base_url(''), name=self.familyname, dointerwiki='y') + url=self.site.base_url(''), name=self.familyname, + dointerwiki=answer)
def test_initial_attributes(self): """Test initial FamilyFileGenerator attributes.""" self.assertEqual(self.generator_instance.base_url, self.site.base_url('')) self.assertEqual(self.generator_instance.name, self.familyname) - self.assertEqual(self.generator_instance.dointerwiki, 'y') + self.assertIn(self.generator_instance.dointerwiki, ['s', 'y']) self.assertIsInstance(self.generator_instance.wikis, dict) self.assertIsInstance(self.generator_instance.langs, list)
@@ -80,9 +83,7 @@ with self.subTest(test='Test element counts'): if self.site.lang not in gen.prefixes: gen.prefixes.append(self.site.lang) - obsolete = self.site.family.interwiki_removals - self.assertCountEqual(set(gen.prefixes) - obsolete, - set(gen.wikis) - obsolete) + self.assertCountEqual(gen.prefixes, gen.wikis)
# test creating Site from url # only test Sites for downloaded wikis (T241413)