jenkins-bot submitted this change.

View Change

[IMPR] Do not kill threads in weblinkchecker.py

- Make count_link_check_threads a staticmethod of WeblinkCheckerRobot
- call suggest_help only once to run the bot and check for
  missing_generator and missing_dependencies
- move all code from finally clause into teardown method
- remove the wait_time and do not kill the threads but ask for
  killing the jobs if KeyboardInterrupt is raised

Bug: T113139
Change-Id: I2f49f05c3622f1c1b21a4dd6d4da29996d0b7034
---
M scripts/weblinkchecker.py
1 file changed, 53 insertions(+), 56 deletions(-)

diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py
index 15838b1..1fd5f6b 100755
--- a/scripts/weblinkchecker.py
+++ b/scripts/weblinkchecker.py
@@ -137,8 +137,9 @@
 try:
     import memento_client
     from memento_client.memento_client import MementoClientException
-except ImportError as e:
-    memento_client = e
+    missing_dependencies = None
+except ImportError:
+    missing_dependencies = ['memento_client']
 
 
 docuReplacements = {'&params;': pagegenerators.parameterHelp}  # noqa: N816
@@ -173,9 +174,6 @@
 
 def _get_closest_memento_url(url, when=None, timegate_uri=None):
     """Get most recent memento for url."""
-    if isinstance(memento_client, ImportError):
-        raise memento_client
-
     if not when:
         when = datetime.datetime.now()
 
@@ -606,6 +604,52 @@
                 thread.daemon = True
                 self.threads.append(thread)
 
+    def teardown(self) -> None:
+        """Finish remaining threads and save history file."""
+        num = self.count_link_check_threads()
+        if num:
+            pywikibot.info('<<lightblue>>Waiting for remaining {} threads '
+                           'to finish, please wait...'.format(num))
+
+        while self.count_link_check_threads():
+            try:
+                time.sleep(0.1)
+            except KeyboardInterrupt:
+                # Threads will die automatically because they are daemonic.
+                if pywikibot.input_yn('There are {} pages remaining in the '
+                                      'queue. Really exit?'
+                                      .format(self.count_link_check_threads()),
+                                      default=False, automatic_quit=False):
+                    break
+
+        num = self.count_link_check_threads()
+        if num:
+            pywikibot.info('<<yellow>>>Remaining {} threads will be killed.'
+                           .format(num))
+
+        if self.history.report_thread:
+            self.history.report_thread.shutdown()
+            # wait until the report thread is shut down; the user can
+            # interrupt it by pressing CTRL-C.
+            try:
+                while self.history.report_thread.is_alive():
+                    time.sleep(0.1)
+            except KeyboardInterrupt:
+                pywikibot.info('Report thread interrupted.')
+                self.history.report_thread.kill()
+
+        pywikibot.info('Saving history...')
+        self.history.save()
+
+    @staticmethod
+    def count_link_check_threads() -> int:
+        """Count LinkCheckThread threads.
+
+        :return: number of LinkCheckThread threads
+        """
+        return sum(isinstance(thread, LinkCheckThread)
+                   for thread in threading.enumerate())
+
 
 def RepeatPageGenerator():  # noqa: N802
     """Generator for pages in History."""
@@ -619,19 +663,6 @@
         yield page
 
 
-def count_link_check_threads() -> int:
-    """
-    Count LinkCheckThread threads.
-
-    :return: number of LinkCheckThread threads
-    """
-    i = 0
-    for thread in threading.enumerate():
-        if isinstance(thread, LinkCheckThread):
-            i += 1
-    return i
-
-
 def main(*args: str) -> None:
     """
     Process command line arguments and invoke bot.
@@ -683,46 +714,12 @@
 
     if not gen:
         gen = gen_factory.getCombinedGenerator()
-    if gen:
+
+    if not suggest_help(missing_generator=not gen,
+                        missing_dependencies=missing_dependencies):
         bot = WeblinkCheckerRobot(http_ignores, config.weblink_dead_days,
                                   generator=gen)
-        try:
-            bot.run()
-        except ImportError:
-            suggest_help(missing_dependencies=('memento_client',))
-            return
-        finally:
-            wait_time = 0
-            # Don't wait longer than 30 seconds for threads to finish.
-            while count_link_check_threads() > 0 and wait_time < 30:
-                try:
-                    pywikibot.output('Waiting for remaining {} threads to '
-                                     'finish, please wait...'
-                                     .format(count_link_check_threads()))
-                    # wait 1 second
-                    time.sleep(1)
-                    wait_time += 1
-                except KeyboardInterrupt:
-                    pywikibot.output('Interrupted.')
-                    break
-            if count_link_check_threads() > 0:
-                pywikibot.output('Remaining {} threads will be killed.'
-                                 .format(count_link_check_threads()))
-                # Threads will die automatically because they are daemonic.
-            if bot.history.report_thread:
-                bot.history.report_thread.shutdown()
-                # wait until the report thread is shut down; the user can
-                # interrupt it by pressing CTRL-C.
-                try:
-                    while bot.history.report_thread.is_alive():
-                        time.sleep(0.1)
-                except KeyboardInterrupt:
-                    pywikibot.output('Report thread interrupted.')
-                    bot.history.report_thread.kill()
-            pywikibot.output('Saving history...')
-            bot.history.save()
-    else:
-        suggest_help(missing_generator=True)
+        bot.run()
 
 
 if __name__ == '__main__':

To view, visit change 789854. To unsubscribe, or for help writing mail filters, visit settings.