http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10283

Revision: 10283 Author: valhallasw Date: 2012-06-03 11:43:31 +0000 (Sun, 03 Jun 2012) Log Message: ----------- + CachedRequest for often-used queries (i.e. namespaces)

Modified Paths: -------------- branches/rewrite/pywikibot/data/api.py

Added Paths: ----------- branches/rewrite/tests/dry_api_tests.py

Modified: branches/rewrite/pywikibot/data/api.py =================================================================== --- branches/rewrite/pywikibot/data/api.py 2012-06-03 11:13:46 UTC (rev 10282) +++ branches/rewrite/pywikibot/data/api.py 2012-06-03 11:43:31 UTC (rev 10283) @@ -379,7 +379,65 @@ # double the next wait, but do not exceed 120 seconds self.retry_wait = min(120, self.retry_wait * 2)

+import datetime +import hashlib +import pickle +import os

+class CachedRequest(Request): + def __init__(self, expiry, *args, **kwargs): + """ expiry should be either a number of days or a datetime.timedelta object """ + super(CachedRequest, self).__init__(*args, **kwargs) + if not isinstance(expiry, datetime.timedelta): + expiry = datetime.timedelta(expiry) + self.expiry = expiry + self._data = None + self._cachetime = None + + def _get_cache_dir(self): + path = os.path.join(pywikibot.config2.base_dir, 'apicache') + self._make_dir(path) + return path + + def _make_dir(self, dir): + try: + os.makedirs(dir) + except OSError: + # directory already exists + pass + + def _create_file_name(self): + return hashlib.sha256(self.http_params()).hexdigest() + + def _cachefile_path(self): + return os.path.join(self._get_cache_dir(), self._create_file_name()) + + def _expired(self, dt): + return dt + self.expiry < datetime.datetime.now() + + def _load_cache(self): + """ Returns whether the cache can be used """ + try: + self._data, self._cachetime = pickle.load(open(self._cachefile_path())) + if self._expired(self._cachetime): + self._data = None + return False + return True + except Exception: + return False + + def _write_cache(self, data): + """ writes data to self._cachefile_path() """ + data = [data, datetime.datetime.now()] + pickle.dump(data, open(self._cachefile_path(), 'w')) + + def submit(self): + cached_available = self._load_cache() + if not cached_available: + self._data = super(CachedRequest, self).submit() + self._write_cache(self._data) + return self._data + class QueryGenerator(object): """Base class for iterators that handle responses to API action=query.

Added: branches/rewrite/tests/dry_api_tests.py =================================================================== --- branches/rewrite/tests/dry_api_tests.py (rev 0) +++ branches/rewrite/tests/dry_api_tests.py 2012-06-03 11:43:31 UTC (rev 10283) @@ -0,0 +1,36 @@ +import datetime +import pywikibot +from pywikibot.data.api import CachedRequest + +parms = {'site': pywikibot.getSite(), + 'action': 'query', + 'meta': 'userinfo'} + +req = CachedRequest(expiry=1, **parms) +expreq = CachedRequest(expiry=0, **parms) +diffreq = CachedRequest(expiry=1, action='query', meta='siteinfo') + +def test_expiry_formats(): + import datetime + assert(req.expiry == CachedRequest(datetime.timedelta(days=1), **parms).expiry) + +def test_get_cache_dir(): + retval = req._get_cache_dir() + assert('apicache' in retval) + +def test_create_file_name(): + assert(req._create_file_name() == req._create_file_name()) + assert(req._create_file_name() == expreq._create_file_name()) + assert(req._create_file_name() != diffreq._create_file_name()) + +def test_cachefile_path(): + assert(req._cachefile_path() == req._cachefile_path()) + assert(req._cachefile_path() == expreq._cachefile_path()) + assert(req._cachefile_path() != diffreq._cachefile_path()) + +def test_expired(): + assert(not req._expired(datetime.datetime.now())) + assert(req._expired(datetime.datetime.now() - datetime.timedelta(days=2))) + + + \ No newline at end of file