jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/826872 )
Change subject: [IMPR] Run login.py script in parallel tasks if -async option is given
......................................................................
[IMPR] Run login.py script in parallel tasks if -async option is given
Change-Id: I3445ab81ca5ce840d4eea7c2a5c259204cced815
---
M pywikibot/scripts/login.py
1 file changed, 52 insertions(+), 27 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/scripts/login.py b/pywikibot/scripts/login.py
index e45273b..8f0900f 100755
--- a/pywikibot/scripts/login.py
+++ b/pywikibot/scripts/login.py
@@ -30,6 +30,8 @@
.. note:: the global account must exist already before
using this.
+ -async Run the bot in parallel tasks
+
If not given as parameter, the script will ask for your username and
password (password entry will be hidden), log in to your home wiki using
this combination, and store the resulting cookies (containing your password
@@ -49,9 +51,13 @@
#
# Distributed under the terms of the MIT license.
#
+import datetime
+from contextlib import suppress
+from concurrent.futures import ThreadPoolExecutor
+
import pywikibot
from pywikibot import config
-from pywikibot.backports import Tuple
+from pywikibot.backports import Tuple, nullcontext
from pywikibot.exceptions import SiteDefinitionError
from pywikibot.login import OauthLoginManager
@@ -90,6 +96,34 @@
oauth_token=oauth_token))
+def login_one_site(code, family, oauth, logout, autocreate):
+ """Login on one site."""
+ try:
+ site = pywikibot.Site(code, family)
+ except SiteDefinitionError:
+ pywikibot.error('{}:{} is not a valid site, '
+ 'please remove it from your user-config'
+ .format(family, code))
+ return
+
+ if oauth:
+ _oauth_login(site)
+ return
+
+ if logout:
+ site.logout()
+ else:
+ site.login(autocreate=autocreate)
+
+ user = site.user()
+ if user:
+ pywikibot.info('Logged in on {} as {}.'.format(site, user))
+ elif logout:
+ pywikibot.info('Logged out of {}.'.format(site))
+ else:
+ pywikibot.info('Not logged in on {}.'.format(site))
+
+
def main(*args: str) -> None:
"""
Process command line arguments and invoke bot.
@@ -102,6 +136,7 @@
logout = False
oauth = False
autocreate = False
+ asyncronous = False
unknown_args = []
for arg in pywikibot.handle_args(args):
if arg == '-all':
@@ -112,6 +147,8 @@
oauth = True
elif arg == '-autocreate':
autocreate = True
+ elif arg == '-async':
+ asyncronous = True
else:
unknown_args += [arg]
@@ -123,33 +160,21 @@
else:
site = pywikibot.Site()
namedict = {site.family.name: {site.code: None}}
- for family_name in namedict:
- for lang in namedict[family_name]:
- try:
- site = pywikibot.Site(code=lang, fam=family_name)
- except SiteDefinitionError:
- pywikibot.output('{}:{} is not a valid site, '
- 'please remove it from your user-config'
- .format(family_name, lang))
- continue
- if oauth:
- _oauth_login(site)
- continue
-
- if logout:
- site.logout()
- else:
- site.login(autocreate=autocreate)
-
- user = site.user()
- if user:
- pywikibot.output('Logged in on {} as {}.'.format(site, user))
- elif logout:
- pywikibot.output('Logged out of {}.'.format(site))
- else:
- pywikibot.output('Not logged in on {}.'.format(site))
+ params = oauth, logout, autocreate
+ context = ThreadPoolExecutor if asyncronous else nullcontext
+ with context() as executor:
+ for family_name in namedict:
+ for lang in namedict[family_name]:
+ if asyncronous:
+ executor.submit(login_one_site, lang, family_name, *params)
+ else:
+ login_one_site(lang, family_name, *params)
if __name__ == '__main__':
- main()
+ start = datetime.datetime.now()
+ with suppress(KeyboardInterrupt):
+ main()
+ pywikibot.info('\nExecution time: {} seconds'
+ .format((datetime.datetime.now() - start).seconds))
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/826872
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I3445ab81ca5ce840d4eea7c2a5c259204cced815
Gerrit-Change-Number: 826872
Gerrit-PatchSet: 2
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged
jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/817190 )
Change subject: [IMPR] process pages in parallel tasks with -async option
......................................................................
[IMPR] process pages in parallel tasks with -async option
This implementation is experimental yet. Processing time is about
7 times faster depending on number of CPU kernels.
Note: The script cannot be interrupted after it is started with this option
Change-Id: I530640292c38890595197441f480f17b82fa4254
---
M scripts/archivebot.py
1 file changed, 18 insertions(+), 5 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/archivebot.py b/scripts/archivebot.py
index c0e2f37..13f250e 100755
--- a/scripts/archivebot.py
+++ b/scripts/archivebot.py
@@ -105,13 +105,15 @@
-keep Preserve thread order in archive even if threads are
archived later
-sort Sort archive by timestamp; should not be used with -keep
+ -async Run the bot in parallel tasks. This is experimental
+ and the bot cannot be stopped with KeyboardInterrupt
.. versionchanged:: 7.6
Localized variables for "archive" template parameter are supported.
`User:MiszaBot/config` is the default template. `-keep` option was
added.
.. versionchanged:: 7.7
- `-sort` option was added.
+ `-sort` and `-async` options were added.
"""
#
# (C) Pywikibot team, 2006-2022
@@ -124,6 +126,7 @@
import re
import time
from collections import OrderedDict, defaultdict
+from concurrent.futures import ThreadPoolExecutor
from hashlib import md5
from math import ceil
from textwrap import fill
@@ -132,7 +135,7 @@
import pywikibot
from pywikibot import i18n
-from pywikibot.backports import List, Set, Tuple, pairwise
+from pywikibot.backports import List, Set, Tuple, nullcontext, pairwise
from pywikibot.exceptions import Error, NoPageError
from pywikibot.textlib import (
TimeStripper,
@@ -850,6 +853,7 @@
calc = None
keep = False
sort = False
+ asyncronous = False
templates = []
local_args = pywikibot.handle_args(args)
@@ -883,6 +887,8 @@
keep = True
elif option == 'sort':
sort = True
+ elif option == 'async':
+ asyncronous = True
site = pywikibot.Site()
@@ -913,15 +919,22 @@
elif pagename:
gen = [pywikibot.Page(site, pagename, ns=3)]
else:
+
ns = [str(namespace)] if namespace is not None else []
pywikibot.output('Fetching template transclusions...')
gen = tmpl.getReferences(only_template_inclusion=True,
follow_redirects=False,
namespaces=ns,
content=True)
- for pg in gen:
- if not process_page(pg, tmpl, salt, force, keep, sort):
- return
+
+ botargs = tmpl, salt, force, keep, sort
+ context = ThreadPoolExecutor if asyncronous else nullcontext
+ with context() as executor:
+ for pg in gen:
+ if asyncronous:
+ executor.submit(process_page, pg, *botargs)
+ elif not process_page(pg, *botargs):
+ return
if __name__ == '__main__':
--
To view, visit https://gerrit.wikimedia.org/r/c/pywikibot/core/+/817190
To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I530640292c38890595197441f480f17b82fa4254
Gerrit-Change-Number: 817190
Gerrit-PatchSet: 11
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: D3r1ck01 <xsavitar.wiki(a)aol.com>
Gerrit-Reviewer: PotsdamLamb
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged