jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/788701 )
Change subject: [bugfix] Preserve more workers than families are handled
......................................................................
[bugfix] Preserve more workers than families are handled
If worker is less than families are handled the script blocks because
no future can be added in preload_family function because all workers
are busy already.
Change-Id: Id6a6819271dcf51f6bb13327dd65e62303f6b668
---
M scripts/maintenance/preload_sites.py
1 file changed, 20 insertions(+), 5 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/maintenance/preload_sites.py b/scripts/maintenance/preload_sites.py
index 7e5b051..3c0afcd 100755
--- a/scripts/maintenance/preload_sites.py
+++ b/scripts/maintenance/preload_sites.py
@@ -16,18 +16,21 @@
"""
#
-# (C) Pywikibot team, 2022
+# (C) Pywikibot team, 2021-2022
#
# Distributed under the terms of the MIT license.
#
+import os
from concurrent.futures import ThreadPoolExecutor, wait
from datetime import datetime
+from typing import Optional, Union
import pywikibot
+from pywikibot.backports import List, Set
from pywikibot.family import Family
-# supported families by this script
+#: supported families by this script
families_list = [
'wikibooks',
'wikinews',
@@ -43,7 +46,7 @@
}
-def preload_family(family, executor):
+def preload_family(family: str, executor: ThreadPoolExecutor) -> None:
"""Preload all sites of a single family file."""
msg = 'Preloading sites of {} family{}'
pywikibot.output(msg.format(family, '...'))
@@ -64,9 +67,21 @@
pywikibot.output(msg.format(family, ' completed.'))
-def preload_families(families, worker):
- """Preload all sites of all given family files."""
+def preload_families(families: Union[List[str], Set[str]],
+ worker: Optional[int]) -> None:
+ """Preload all sites of all given family files.
+
+ .. versionchanged:: 7.3
+ Default of worker is calculated like for Python 3.8 but preserves
+ at least one worker more than families_list elements to ensure a
+ worker can be added in :func:`preload_family`.
+ """
start = datetime.now()
+ if worker is None:
+ # Python 3.8 default
+ worker = min(32, (os.cpu_count() or 1) + 4)
+ # allow to add futures in preload_family
+ worker = max(len(families) + 1, worker)
with ThreadPoolExecutor(worker) as executor:
futures = {executor.submit(preload_family, family, executor)
for family in families}
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/788701
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: Id6a6819271dcf51f6bb13327dd65e62303f6b668
Gerrit-Change-Number: 788701
Gerrit-PatchSet: 6
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: D3r1ck01 <xsavitar.wiki(a)aol.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged