http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10816

Revision: 10816 Author: btongminh Date: 2012-12-23 17:07:01 +0000 (Sun, 23 Dec 2012) Log Message: ----------- * Add max_runtime config setting to CommonsDelinker * Fix r10814

Modified Paths: -------------- trunk/pywikipedia/commonsdelinker/delinker.py trunk/pywikipedia/commonsdelinker/image_replacer.py trunk/pywikipedia/commonsdelinker/threadpool.py

Modified: trunk/pywikipedia/commonsdelinker/delinker.py =================================================================== --- trunk/pywikipedia/commonsdelinker/delinker.py 2012-12-23 16:01:12 UTC (rev 10815) +++ trunk/pywikipedia/commonsdelinker/delinker.py 2012-12-23 17:07:01 UTC (rev 10816) @@ -849,6 +849,8 @@ self.database.commit()

def start(self): + start_time = time.time() + # Gracefully exit all threads on SIG_INT or SIG_TERM threadpool.catch_signals()

@@ -871,7 +873,21 @@ self.read_deletion_log() if self.config.get('enable_replacer', False): self.read_replacement_log() - + + if 'max_runtime' in self.config: + if (self.config['max_runtime'] + start_time) < time.time(): + output(u'Maximum run time exceeded; trying to shutdown') + while not (self.CheckUsages.is_idle() and + self.Delinkers.is_idle() and + self.Loggers.is_idle()): + time.sleep(self.config['timeout']) + + self.CheckUsages.exit() + self.Delinkers.exit() + self.Loggers.exit() + output(u'All work done; exiting') + return + time.sleep(self.config['timeout'])

def thread_died(self):

Modified: trunk/pywikipedia/commonsdelinker/image_replacer.py =================================================================== --- trunk/pywikipedia/commonsdelinker/image_replacer.py 2012-12-23 16:01:12 UTC (rev 10815) +++ trunk/pywikipedia/commonsdelinker/image_replacer.py 2012-12-23 17:07:01 UTC (rev 10816) @@ -247,7 +247,7 @@ report = self.reporters[0] del self.reporters[0]

- self.report(*report) + self.report(report)

def report(self, (old_image, new_image, user, comment, not_ok)): not_ok_items = [] @@ -316,6 +316,7 @@ output('A critical error has occured! Aborting!') traceback.print_exc(file = sys.stderr) finally: + output('Exitting replacer') wikipedia.stopme()

if __name__ == '__main__': main()

Modified: trunk/pywikipedia/commonsdelinker/threadpool.py =================================================================== --- trunk/pywikipedia/commonsdelinker/threadpool.py 2012-12-23 16:01:12 UTC (rev 10815) +++ trunk/pywikipedia/commonsdelinker/threadpool.py 2012-12-23 17:07:01 UTC (rev 10816) @@ -86,12 +86,23 @@ finally: self.jobLock.release()

+ def is_idle(self): + self.jobLock.acquire() + try: + idle = len(self.jobQueue) == 0 + for thread in self.threads: + idle = idle and thread.is_idle + finally: + self.jobLock.release() + return idle + class Thread(threading.Thread): timeout = None def __init__(self, pool): threading.Thread.__init__(self) self.pool = pool self.quit = False + self.is_idle = True

def run(self): while True: @@ -118,9 +129,11 @@ if self.starve(): return continue job = self.pool.jobQueue.pop(0) + self.is_idle = False self.pool.jobLock.release()

self.do(job) + self.is_idle = True

def exit(self): self.pool.jobLock.acquire()