Revision: 5252
Author: russblau
Date: 2008-04-22 20:15:23 +0000 (Tue, 22 Apr 2008)
Log Message:
-----------
Use throttle to handle server lag delays; fix bug in api request formation.
Modified Paths:
--------------
branches/rewrite/pywikibot/data/api.py
branches/rewrite/pywikibot/throttle.py
Modified: branches/rewrite/pywikibot/data/api.py
===================================================================
--- branches/rewrite/pywikibot/data/api.py 2008-04-22 11:46:44 UTC (rev 5251)
+++ branches/rewrite/pywikibot/data/api.py 2008-04-22 20:15:23 UTC (rev 5252)
@@ -141,31 +141,28 @@
if self.params['format'] != 'json':
raise TypeError("Query format '%s' cannot be parsed."
% self.params['format'])
- uri = self.site.scriptpath() + "/api.php"
for key in self.params:
if isinstance(self.params[key], unicode):
self.params[key] = self.params[key].encode(self.site.encoding())
params = urllib.urlencode(self.params)
while True:
# TODO catch http errors
+ self.site.throttle() # TODO: add write=True when needed
+ uri = self.site.scriptpath() + "/api.php"
try:
- self.site.sitelock.acquire()
- try:
- if self.params.get("action", "") in
("login",):
- rawdata = http.request(self.site, uri, method="POST",
- headers={'Content-Type':
-
'application/x-www-form-urlencoded'},
- body=params)
- else:
- uri = uri + "?" + params
- rawdata = http.request(self.site, uri)
- except Exception, e: #TODO: what exceptions can occur here?
- logging.warning(traceback.format_exc())
- print uri, params
- self.wait()
- continue
- finally:
- self.site.sitelock.release()
+ if self.params.get("action", "") in
("login",):
+ rawdata = http.request(self.site, uri, method="POST",
+ headers={'Content-Type':
+ 'application/x-www-form-urlencoded'},
+ body=params)
+ else:
+ uri = uri + "?" + params
+ rawdata = http.request(self.site, uri)
+ except Exception, e: #TODO: what exceptions can occur here?
+ logging.warning(traceback.format_exc())
+ print uri, params
+ self.wait()
+ continue
if rawdata.startswith(u"unknown_action"):
raise APIError(rawdata[:14], rawdata[16:])
try:
@@ -205,7 +202,7 @@
if lag:
logging.info(
"Pausing due to database lag: " + info)
- self.lag_wait(int(lag.group("lag")))
+ self.site.throttle.lag(int(lag.group("lag")))
continue
if code in (u'internal_api_error_DBConnectionError', ):
self.wait()
@@ -443,7 +440,7 @@
"""
Login to the site.
- Paramters are all ignored.
+ Parameters are all ignored.
Returns cookie data if succesful, None otherwise.
"""
@@ -477,11 +474,14 @@
if __name__ == "__main__":
from pywikibot import Site
+ logging.getLogger().setLevel(logging.DEBUG)
mysite = Site("en", "wikipedia")
- logging.getLogger().setLevel(logging.DEBUG)
+ print "starting test...."
def _test():
import doctest
doctest.testmod()
- _test()
- pywikibot.stopme()
+ try:
+ _test()
+ finally:
+ pywikibot.stopme()
Modified: branches/rewrite/pywikibot/throttle.py
===================================================================
--- branches/rewrite/pywikibot/throttle.py 2008-04-22 11:46:44 UTC (rev 5251)
+++ branches/rewrite/pywikibot/throttle.py 2008-04-22 20:15:23 UTC (rev 5252)
@@ -229,3 +229,23 @@
finally:
self.lock.release()
+ def lag(self, lagtime):
+ """
+ Seize the throttle lock due to server lag.
+
+ This will prevent any thread from accessing this site.
+
+ """
+ started = time.time()
+ self.lock.acquire()
+ try:
+ # start at 1/2 the current server lag time
+ # wait at least 5 seconds but not more than 120 seconds
+ delay = min(max(5, lagtime//2), 120)
+ # account for any time we waited while acquiring the lock
+ wait = delay - (time.time() - started)
+ if wait > 0:
+ time.sleep(wait)
+ finally:
+ self.lock.release()
+