Revision: 5275
Author: btongminh
Date: 2008-04-26 12:05:51 +0000 (Sat, 26 Apr 2008)
Log Message:
-----------
Add starvation to logger threads. Remove obsolete code.
Modified Paths:
--------------
trunk/pywikipedia/commonsdelinker/delinker.py
Modified: trunk/pywikipedia/commonsdelinker/delinker.py
===================================================================
--- trunk/pywikipedia/commonsdelinker/delinker.py 2008-04-25 19:13:20 UTC (rev 5274)
+++ trunk/pywikipedia/commonsdelinker/delinker.py 2008-04-26 12:05:51 UTC (rev 5275)
@@ -535,6 +535,8 @@
self.pool.jobLock.release()
class Logger(threadpool.Thread):
+ timeout = 360
+
def __init__(self, pool, CommonsDelinker):
threadpool.Thread.__init__(self, pool)
self.CommonsDelinker = CommonsDelinker
@@ -607,7 +609,21 @@
traceback.print_exc(file = sys.stderr)
self.exit()
self.CommonsDelinker.thread_died()
+
+ def starve(self):
+ self.pool.jobLock.acquire()
+ try:
+ if self.pool[id(self)].isSet(): return False
+ output(u'%s Starving' % self)
+ self.database.close()
+ del self.pool[id(self)]
+ self.pool.threads.remove(self)
+ return True
+ finally:
+ self.pool.jobLock.release()
+
+
class CommonsDelinker(object):
def __init__(self):
self.config = config.CommonsDelinker
@@ -832,33 +848,8 @@
time.sleep(self.config['timeout'])
def thread_died(self):
- # A thread died, it may be possible that we cannot
- # function any more. Currently only for CheckUsages
- # and Loggers. Delinkers should not be able to die.
- cu = 0
- self.CheckUsages.jobLock.acquire()
- for thread in self.CheckUsages.threads:
- if thread.isAlive() and not thread.quit:
- cu += 1
- self.CheckUsages.jobLock.release()
- lg = 0
- self.Loggers.jobLock.acquire()
- for thread in self.Loggers.threads:
- if thread.isAlive() and not thread.quit:
- lg += 1
- unlogged = self.Loggers.jobQueue[:]
- self.Loggers.jobLock.release()
-
- # We can no longer function if we have only one
- # CheckUsage or zero Loggers available.
- # TODO: config settings?
- if cu <= 1:
- output(u'ERROR!!! Too few CheckUsages left to function', False)
- threadpool.terminate()
- if lg <= 0:
- output(u'ERROR!!! Too few Loggers left to function', False)
- print >>sys.stderr, 'Currently unlogged:', unlogged
- threadpool.terminate()
+ # Obsolete
+ return
@staticmethod
def output(*args):