jenkins-bot has submitted this change. ( https://gerrit.wikimedia.org/r/c/pywikibot/core/+/789854 )

Change subject: [IMPR] Do not kill threads in weblinkchecker.py ......................................................................

[IMPR] Do not kill threads in weblinkchecker.py

- Make count_link_check_threads a staticmethod of WeblinkCheckerRobot - call suggest_help only once to run the bot and check for missing_generator and missing_dependencies - move all code from finally clause into teardown method - remove the wait_time and do not kill the threads but ask for killing the jobs if KeyboardInterrupt is raised

Bug: T113139 Change-Id: I2f49f05c3622f1c1b21a4dd6d4da29996d0b7034 --- M scripts/weblinkchecker.py 1 file changed, 53 insertions(+), 56 deletions(-)

Approvals: Rubin: Looks good to me, but someone else must approve Xqt: Looks good to me, approved jenkins-bot: Verified

diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py index 15838b1..1fd5f6b 100755 --- a/scripts/weblinkchecker.py +++ b/scripts/weblinkchecker.py @@ -137,8 +137,9 @@ try: import memento_client from memento_client.memento_client import MementoClientException -except ImportError as e: - memento_client = e + missing_dependencies = None +except ImportError: + missing_dependencies = ['memento_client']

docuReplacements = {'&params;': pagegenerators.parameterHelp} # noqa: N816 @@ -173,9 +174,6 @@

def _get_closest_memento_url(url, when=None, timegate_uri=None): """Get most recent memento for url.""" - if isinstance(memento_client, ImportError): - raise memento_client - if not when: when = datetime.datetime.now()

@@ -606,6 +604,52 @@ thread.daemon = True self.threads.append(thread)

+ def teardown(self) -> None: + """Finish remaining threads and save history file.""" + num = self.count_link_check_threads() + if num: + pywikibot.info('<<lightblue>>Waiting for remaining {} threads ' + 'to finish, please wait...'.format(num)) + + while self.count_link_check_threads(): + try: + time.sleep(0.1) + except KeyboardInterrupt: + # Threads will die automatically because they are daemonic. + if pywikibot.input_yn('There are {} pages remaining in the ' + 'queue. Really exit?' + .format(self.count_link_check_threads()), + default=False, automatic_quit=False): + break + + num = self.count_link_check_threads() + if num: + pywikibot.info('<<yellow>>>Remaining {} threads will be killed.' + .format(num)) + + if self.history.report_thread: + self.history.report_thread.shutdown() + # wait until the report thread is shut down; the user can + # interrupt it by pressing CTRL-C. + try: + while self.history.report_thread.is_alive(): + time.sleep(0.1) + except KeyboardInterrupt: + pywikibot.info('Report thread interrupted.') + self.history.report_thread.kill() + + pywikibot.info('Saving history...') + self.history.save() + + @staticmethod + def count_link_check_threads() -> int: + """Count LinkCheckThread threads. + + :return: number of LinkCheckThread threads + """ + return sum(isinstance(thread, LinkCheckThread) + for thread in threading.enumerate()) +

def RepeatPageGenerator(): # noqa: N802 """Generator for pages in History.""" @@ -619,19 +663,6 @@ yield page

-def count_link_check_threads() -> int: - """ - Count LinkCheckThread threads. - - :return: number of LinkCheckThread threads - """ - i = 0 - for thread in threading.enumerate(): - if isinstance(thread, LinkCheckThread): - i += 1 - return i - - def main(*args: str) -> None: """ Process command line arguments and invoke bot. @@ -683,46 +714,12 @@

if not gen: gen = gen_factory.getCombinedGenerator() - if gen: + + if not suggest_help(missing_generator=not gen, + missing_dependencies=missing_dependencies): bot = WeblinkCheckerRobot(http_ignores, config.weblink_dead_days, generator=gen) - try: - bot.run() - except ImportError: - suggest_help(missing_dependencies=('memento_client',)) - return - finally: - wait_time = 0 - # Don't wait longer than 30 seconds for threads to finish. - while count_link_check_threads() > 0 and wait_time < 30: - try: - pywikibot.output('Waiting for remaining {} threads to ' - 'finish, please wait...' - .format(count_link_check_threads())) - # wait 1 second - time.sleep(1) - wait_time += 1 - except KeyboardInterrupt: - pywikibot.output('Interrupted.') - break - if count_link_check_threads() > 0: - pywikibot.output('Remaining {} threads will be killed.' - .format(count_link_check_threads())) - # Threads will die automatically because they are daemonic. - if bot.history.report_thread: - bot.history.report_thread.shutdown() - # wait until the report thread is shut down; the user can - # interrupt it by pressing CTRL-C. - try: - while bot.history.report_thread.is_alive(): - time.sleep(0.1) - except KeyboardInterrupt: - pywikibot.output('Report thread interrupted.') - bot.history.report_thread.kill() - pywikibot.output('Saving history...') - bot.history.save() - else: - suggest_help(missing_generator=True) + bot.run()

if __name__ == '__main__':