jenkins-bot has submitted this change. (
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/789854 )
Change subject: [IMPR] Do not kill threads in weblinkchecker.py
......................................................................
[IMPR] Do not kill threads in weblinkchecker.py
- Make count_link_check_threads a staticmethod of WeblinkCheckerRobot
- call suggest_help only once to run the bot and check for
missing_generator and missing_dependencies
- move all code from finally clause into teardown method
- remove the wait_time and do not kill the threads but ask for
killing the jobs if KeyboardInterrupt is raised
Bug: T113139
Change-Id: I2f49f05c3622f1c1b21a4dd6d4da29996d0b7034
---
M scripts/weblinkchecker.py
1 file changed, 53 insertions(+), 56 deletions(-)
Approvals:
Rubin: Looks good to me, but someone else must approve
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py
index 15838b1..1fd5f6b 100755
--- a/scripts/weblinkchecker.py
+++ b/scripts/weblinkchecker.py
@@ -137,8 +137,9 @@
try:
import memento_client
from memento_client.memento_client import MementoClientException
-except ImportError as e:
- memento_client = e
+ missing_dependencies = None
+except ImportError:
+ missing_dependencies = ['memento_client']
docuReplacements = {'¶ms;': pagegenerators.parameterHelp} # noqa: N816
@@ -173,9 +174,6 @@
def _get_closest_memento_url(url, when=None, timegate_uri=None):
"""Get most recent memento for url."""
- if isinstance(memento_client, ImportError):
- raise memento_client
-
if not when:
when = datetime.datetime.now()
@@ -606,6 +604,52 @@
thread.daemon = True
self.threads.append(thread)
+ def teardown(self) -> None:
+ """Finish remaining threads and save history
file."""
+ num = self.count_link_check_threads()
+ if num:
+ pywikibot.info('<<lightblue>>Waiting for remaining {} threads
'
+ 'to finish, please wait...'.format(num))
+
+ while self.count_link_check_threads():
+ try:
+ time.sleep(0.1)
+ except KeyboardInterrupt:
+ # Threads will die automatically because they are daemonic.
+ if pywikibot.input_yn('There are {} pages remaining in the '
+ 'queue. Really exit?'
+ .format(self.count_link_check_threads()),
+ default=False, automatic_quit=False):
+ break
+
+ num = self.count_link_check_threads()
+ if num:
+ pywikibot.info('<<yellow>>>Remaining {} threads will be
killed.'
+ .format(num))
+
+ if self.history.report_thread:
+ self.history.report_thread.shutdown()
+ # wait until the report thread is shut down; the user can
+ # interrupt it by pressing CTRL-C.
+ try:
+ while self.history.report_thread.is_alive():
+ time.sleep(0.1)
+ except KeyboardInterrupt:
+ pywikibot.info('Report thread interrupted.')
+ self.history.report_thread.kill()
+
+ pywikibot.info('Saving history...')
+ self.history.save()
+
+ @staticmethod
+ def count_link_check_threads() -> int:
+ """Count LinkCheckThread threads.
+
+ :return: number of LinkCheckThread threads
+ """
+ return sum(isinstance(thread, LinkCheckThread)
+ for thread in threading.enumerate())
+
def RepeatPageGenerator(): # noqa: N802
"""Generator for pages in History."""
@@ -619,19 +663,6 @@
yield page
-def count_link_check_threads() -> int:
- """
- Count LinkCheckThread threads.
-
- :return: number of LinkCheckThread threads
- """
- i = 0
- for thread in threading.enumerate():
- if isinstance(thread, LinkCheckThread):
- i += 1
- return i
-
-
def main(*args: str) -> None:
"""
Process command line arguments and invoke bot.
@@ -683,46 +714,12 @@
if not gen:
gen = gen_factory.getCombinedGenerator()
- if gen:
+
+ if not suggest_help(missing_generator=not gen,
+ missing_dependencies=missing_dependencies):
bot = WeblinkCheckerRobot(http_ignores, config.weblink_dead_days,
generator=gen)
- try:
- bot.run()
- except ImportError:
- suggest_help(missing_dependencies=('memento_client',))
- return
- finally:
- wait_time = 0
- # Don't wait longer than 30 seconds for threads to finish.
- while count_link_check_threads() > 0 and wait_time < 30:
- try:
- pywikibot.output('Waiting for remaining {} threads to '
- 'finish, please wait...'
- .format(count_link_check_threads()))
- # wait 1 second
- time.sleep(1)
- wait_time += 1
- except KeyboardInterrupt:
- pywikibot.output('Interrupted.')
- break
- if count_link_check_threads() > 0:
- pywikibot.output('Remaining {} threads will be killed.'
- .format(count_link_check_threads()))
- # Threads will die automatically because they are daemonic.
- if bot.history.report_thread:
- bot.history.report_thread.shutdown()
- # wait until the report thread is shut down; the user can
- # interrupt it by pressing CTRL-C.
- try:
- while bot.history.report_thread.is_alive():
- time.sleep(0.1)
- except KeyboardInterrupt:
- pywikibot.output('Report thread interrupted.')
- bot.history.report_thread.kill()
- pywikibot.output('Saving history...')
- bot.history.save()
- else:
- suggest_help(missing_generator=True)
+ bot.run()
if __name__ == '__main__':
--
To view, visit
https://gerrit.wikimedia.org/r/c/pywikibot/core/+/789854
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Change-Id: I2f49f05c3622f1c1b21a4dd6d4da29996d0b7034
Gerrit-Change-Number: 789854
Gerrit-PatchSet: 3
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: D3r1ck01 <xsavitar.wiki(a)aol.com>
Gerrit-Reviewer: Rubin <rubin.happy(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
Gerrit-MessageType: merged