[bugfix] Remove sites from FamilyFileGenerator.langs on error

- If loading a site fails, remove it from langs attribute of
  FamilyFileGenerator
- [s]trict: add the ability to process sites with the same domain only
- with [e]dit sites can be given delimited by space, comma or both
- collect self.prefixes for tests after FamilyFileGenerator.getapis()
  was processed
- use [s]trict option for Wikimedia site for within tests
- remove old implementation removing obsolete sites in tests

Bug: T334775
Bug: T334714
Change-Id: I5b049ead55e19f2a4c2ba522f34412e541737669
---
M tests/generate_family_file_tests.py
M pywikibot/scripts/generate_family_file.py
2 files changed, 79 insertions(+), 22 deletions(-)

diff --git a/pywikibot/scripts/generate_family_file.py b/pywikibot/scripts/generate_family_file.py
index d5c00cc..8b0607c 100755
--- a/pywikibot/scripts/generate_family_file.py
+++ b/pywikibot/scripts/generate_family_file.py
@@ -12,7 +12,7 @@
 
     <url>:         an url from where the family settings are loaded
     <name>:        the family name without "_family.py" tail.
-    <dointerwiki>: predefined answer (y|n) to add multiple site codes
+    <dointerwiki>: predefined answer (y|s|n) to add multiple site codes
     <verify>:      disable certificate validaton `(y|n)
 
 Example::
@@ -25,14 +25,18 @@
 .. versionchanged:: 7.0
    moved to pywikibot.scripts folder; create family files in families
    folder of your base directory instead of pywikibot/families.
+.. versionchanged:: 8.1
+   [s]trict can be given for <dointerwiki> parameter to ensure that
+   sites are from the given domain.
 """
 #
-# (C) Pywikibot team, 2010-2022
+# (C) Pywikibot team, 2010-2023
 #
 # Distributed under the terms of the MIT license
 #
 import codecs
 import os
+import re
 import string
 import sys
 from contextlib import suppress
@@ -63,8 +67,10 @@
         :param url: an url from where the family settings are loaded
         :param name: the family name without "_family.py" tail.
         :param dointerwiki: Predefined answer to add multiple site
-            codes. Pass `Y` or `y` for yes `N` or `n` for no and
-            `E` or `e` if you want to edit the collection of sites.
+            codes. Pass `Y` or `y` for yes, `S` or `s` for strict which
+            only includes site of the same domain (usually for Wikimedia
+            sites), `N` or `n` for no and `E` or `e` if you want to edit
+            the collection of sites.
         :param verify: If a certificate verification failes, you may
             pass `Y` or `y` to disable certificate validaton `N` or `n`
             to keep it enabled.
@@ -149,7 +155,14 @@
         self.writefile(verify)
 
     def getlangs(self, w) -> None:
-        """Determine site code of a family."""
+        """Determine site code of a family.
+
+        .. versionchanged:: 8.1
+           with [e]dit the interwiki list can be given delimited by
+           space or comma or both. With [s]trict only sites with the
+           same domain are collected. A [h]elp answer was added to show
+           more information about possible answers.
+        """
         print('Determining other sites...', end='')
         try:
             self.langs = w.langs
@@ -169,21 +182,39 @@
         code_len = len(self.langs)
         if code_len > 1:
             if self.dointerwiki is None:
-                makeiw = input(
-                    '\nThere are {} sites available.'
-                    '\nDo you want to generate interwiki links? '
-                    'This might take a long time. ([y]es/[N]o/[e]dit)'
-                    .format(code_len)).lower()
+                while True:
+                    makeiw = input(
+                        '\n'
+                        f'There are {code_len} sites available.'
+                        ' Do you want to generate interwiki links?\n'
+                        'This might take a long time. '
+                        '([y]es, [s]trict, [N]o, [e]dit), [h]elp) ').lower()
+                    if makeiw in ('y', 's', 'n', 'e', ''):
+                        break
+                    print(
+                        '\n'
+                        '[y]es:    create interwiki links for all sites\n'
+                        '[s]trict: yes, but for sites with same domain only\n'
+                        '[N]o:     no, use the current site only (default)\n'
+                        '[e]dit:   get a list delimited with space or comma\n'
+                        '[h]elp:   this help message'
+                    )
             else:
                 makeiw = self.dointerwiki
 
-            if makeiw == 'n':
+            if makeiw in ('n', ''):
                 self.langs = [wiki for wiki in self.langs
                               if wiki['url'] == w.iwpath]
+            elif makeiw == 's':
+                domain = '.'.join(urlparse(w.server).hostname.split('.')[1:])
+                self.langs = [wiki for wiki in self.langs
+                              if domain in wiki['url']]
+
             elif makeiw == 'e':
                 for wiki in self.langs:
                     print(wiki['prefix'], wiki['url'])
-                do_langs = input('Which sites do you want: ')
+                do_langs = re.split(' *,| +',
+                                    input('Which sites do you want: '))
                 self.langs = [wiki for wiki in self.langs
                               if wiki['prefix'] in do_langs
                               or wiki['url'] == w.iwpath]
@@ -196,7 +227,8 @@
 
     def getapis(self) -> None:
         """Load other site pages."""
-        print('Loading wikis... ')
+        print(f'Loading {len(self.langs)} wikis... ')
+        remove = []
         for lang in self.langs:
             key = lang['prefix']
             print(f'  * {key}... ', end='')
@@ -206,17 +238,21 @@
                     print('downloaded')
                 except Exception as e:  # pragma: no cover
                     print(e)
+                    remove.append(lang)
             else:
                 print('in cache')
 
+        for lang in remove:
+            self.langs.remove(lang)
+
     def writefile(self, verify) -> None:
         """Write the family file."""
         fn = os.path.join(self.base_dir, 'families',
                           f'{self.name}_family.py')
         print(f'Writing {fn}... ')
 
-        if os.path.exists(fn) and input('{} already exists. Overwrite? (y/n)'
-                                        .format(fn)).lower() == 'n':
+        if os.path.exists(fn) and input(
+                f'{fn} already exists. Overwrite? (y/n) ').lower() == 'n':
             print('Terminating.')
             sys.exit(1)
 
diff --git a/tests/generate_family_file_tests.py b/tests/generate_family_file_tests.py
index f219ac6..e97f884 100755
--- a/tests/generate_family_file_tests.py
+++ b/tests/generate_family_file_tests.py
@@ -11,6 +11,7 @@
 from urllib.parse import urlparse
 
 from pywikibot import Site
+from pywikibot.family import WikimediaFamily
 from pywikibot.scripts import generate_family_file
 from tests.aspects import DefaultSiteTestCase
 from tests.utils import skipping
@@ -29,15 +30,15 @@
                 self.langs.append(wiki)
                 break
 
-        self.prefixes = [item['prefix'] for item in self.langs]
         super().getapis()
+        self.prefixes = [item['prefix'] for item in self.langs]
         self.langs = save
 
     def writefile(self, verify):
         """Pass writing."""
 
 
-class TestGenerateFamilyFiles(DefaultSiteTestCase):
+class TestGenerateFamilyFile(DefaultSiteTestCase):
 
     """Test generate_family_file functionality."""
 
@@ -55,15 +56,17 @@
     def setUp(self):
         """Set up tests."""
         super().setUp()
+        answer = 's' if isinstance(self.site.family, WikimediaFamily) else 'y'
         self.generator_instance = FamilyTestGenerator(
-            url=self.site.base_url(''), name=self.familyname, dointerwiki='y')
+            url=self.site.base_url(''), name=self.familyname,
+            dointerwiki=answer)
 
     def test_initial_attributes(self):
         """Test initial FamilyFileGenerator attributes."""
         self.assertEqual(self.generator_instance.base_url,
                          self.site.base_url(''))
         self.assertEqual(self.generator_instance.name, self.familyname)
-        self.assertEqual(self.generator_instance.dointerwiki, 'y')
+        self.assertIn(self.generator_instance.dointerwiki, ['s', 'y'])
         self.assertIsInstance(self.generator_instance.wikis, dict)
         self.assertIsInstance(self.generator_instance.langs, list)
 
@@ -80,9 +83,7 @@
             with self.subTest(test='Test element counts'):
                 if self.site.lang not in gen.prefixes:
                     gen.prefixes.append(self.site.lang)
-                obsolete = self.site.family.interwiki_removals
-                self.assertCountEqual(set(gen.prefixes) - obsolete,
-                                      set(gen.wikis) - obsolete)
+                self.assertCountEqual(gen.prefixes, gen.wikis)
 
         # test creating Site from url
         # only test Sites for downloaded wikis (T241413)

