jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/660758 )

Change subject: [IMPR] speed up preload_sites.py maintenance script ......................................................................

[IMPR] speed up preload_sites.py maintenance script

Increase the number of parallel workers. For example siteinfo of all 750 sites can be preloaded in 20 seconds with 25 workers.

Bug: T226157 Change-Id: I9e4f8810f4a969fb58a397f94add52ace293d25b --- M scripts/maintenance/preload_sites.py 1 file changed, 7 insertions(+), 5 deletions(-)

Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified

diff --git a/scripts/maintenance/preload_sites.py b/scripts/maintenance/preload_sites.py index 5eb4f2a..f44aad3 100644 --- a/scripts/maintenance/preload_sites.py +++ b/scripts/maintenance/preload_sites.py @@ -43,7 +43,7 @@ }

-def preload_family(family): +def preload_family(family, executor): """Preload all sites of a single family file.""" msg = 'Preloading sites of {} family{}' pywikibot.output(msg.format(family, '...')) @@ -54,11 +54,13 @@ codes.remove(code) obsolete = Family.load(family).obsolete

+ futures = set() for code in codes: if code not in obsolete: site = pywikibot.Site(code, family) - pywikibot.Page(site, 'Main page') # title does not care - + # page title does not care + futures.add(executor.submit(pywikibot.Page, site, 'Main page')) + wait(futures) pywikibot.output(msg.format(family, ' completed.'))

@@ -66,8 +68,8 @@ """Preload all sites of all given family files.""" start = datetime.now() with ThreadPoolExecutor(worker) as executor: - futures = {executor.submit(preload_family, family): - family for family in families} + futures = {executor.submit(preload_family, family, executor) + for family in families} wait(futures) pywikibot.output('Loading time used: {}'.format(datetime.now() - start))