Revision: 5252 Author: russblau Date: 2008-04-22 20:15:23 +0000 (Tue, 22 Apr 2008)

Log Message: ----------- Use throttle to handle server lag delays; fix bug in api request formation.

Modified Paths: -------------- branches/rewrite/pywikibot/data/api.py branches/rewrite/pywikibot/throttle.py

Modified: branches/rewrite/pywikibot/data/api.py =================================================================== --- branches/rewrite/pywikibot/data/api.py 2008-04-22 11:46:44 UTC (rev 5251) +++ branches/rewrite/pywikibot/data/api.py 2008-04-22 20:15:23 UTC (rev 5252) @@ -141,31 +141,28 @@ if self.params['format'] != 'json': raise TypeError("Query format '%s' cannot be parsed." % self.params['format']) - uri = self.site.scriptpath() + "/api.php" for key in self.params: if isinstance(self.params[key], unicode): self.params[key] = self.params[key].encode(self.site.encoding()) params = urllib.urlencode(self.params) while True: # TODO catch http errors + self.site.throttle() # TODO: add write=True when needed + uri = self.site.scriptpath() + "/api.php" try: - self.site.sitelock.acquire() - try: - if self.params.get("action", "") in ("login",): - rawdata = http.request(self.site, uri, method="POST", - headers={'Content-Type': - 'application/x-www-form-urlencoded'}, - body=params) - else: - uri = uri + "?" + params - rawdata = http.request(self.site, uri) - except Exception, e: #TODO: what exceptions can occur here? - logging.warning(traceback.format_exc()) - print uri, params - self.wait() - continue - finally: - self.site.sitelock.release() + if self.params.get("action", "") in ("login",): + rawdata = http.request(self.site, uri, method="POST", + headers={'Content-Type': + 'application/x-www-form-urlencoded'}, + body=params) + else: + uri = uri + "?" + params + rawdata = http.request(self.site, uri) + except Exception, e: #TODO: what exceptions can occur here? + logging.warning(traceback.format_exc()) + print uri, params + self.wait() + continue if rawdata.startswith(u"unknown_action"): raise APIError(rawdata[:14], rawdata[16:]) try: @@ -205,7 +202,7 @@ if lag: logging.info( "Pausing due to database lag: " + info) - self.lag_wait(int(lag.group("lag"))) + self.site.throttle.lag(int(lag.group("lag"))) continue if code in (u'internal_api_error_DBConnectionError', ): self.wait() @@ -443,7 +440,7 @@ """ Login to the site.

- Paramters are all ignored. + Parameters are all ignored.

Returns cookie data if succesful, None otherwise. """ @@ -477,11 +474,14 @@

if __name__ == "__main__": from pywikibot import Site + logging.getLogger().setLevel(logging.DEBUG) mysite = Site("en", "wikipedia") - logging.getLogger().setLevel(logging.DEBUG) + print "starting test...." def _test(): import doctest doctest.testmod() - _test() - pywikibot.stopme() + try: + _test() + finally: + pywikibot.stopme()

Modified: branches/rewrite/pywikibot/throttle.py =================================================================== --- branches/rewrite/pywikibot/throttle.py 2008-04-22 11:46:44 UTC (rev 5251) +++ branches/rewrite/pywikibot/throttle.py 2008-04-22 20:15:23 UTC (rev 5252) @@ -229,3 +229,23 @@ finally: self.lock.release()

+ def lag(self, lagtime): + """ + Seize the throttle lock due to server lag. + + This will prevent any thread from accessing this site. + + """ + started = time.time() + self.lock.acquire() + try: + # start at 1/2 the current server lag time + # wait at least 5 seconds but not more than 120 seconds + delay = min(max(5, lagtime//2), 120) + # account for any time we waited while acquiring the lock + wait = delay - (time.time() - started) + if wait > 0: + time.sleep(wait) + finally: + self.lock.release() +