[Pywikibot-commits] [Gerrit] Fix cached test running (site_tests.py) - change (pywikibot/core)

jenkins-bot has submitted this change and it was merged. Change subject: Fix cached test running (site_tests.py) ...................................................................... Fix cached test running (site_tests.py) + fix .gitignore to *not* ignore pywikibot/data/* Change-Id: Ib3ff27f5f41f2236eb5510c96751936ac7e98f1e --- M .gitignore M pywikibot/data/api.py M tests/__init__.py M tests/site_tests.py 4 files changed, 49 insertions(+), 16 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/.gitignore b/.gitignore index df24ac5..90d2199 100644 --- a/.gitignore +++ b/.gitignore @@ -5,7 +5,7 @@ *.pyc /.tox /*.egg -data* +/data* logs* *cache* .idea diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index f129665..a12764a 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -35,8 +35,13 @@ from pywikibot.exceptions import * import sys + if sys.version_info[0] > 2: basestring = (str, ) + from urllib.parse import urlencode, unquote + unicode = str +else: + from urllib import urlencode, unquote _logger = "data.api" @@ -174,9 +179,17 @@ return iter(self.params.items()) def http_params(self): - """Return the parameters formatted for inclusion in an HTTP request.""" + """Return the parameters formatted for inclusion in an HTTP request. + + self.params MUST be either + list of unicode + unicode (may be |-separated list) + str in site encoding (may be |-separated list) + """ for key in self.params: + if isinstance(self.params[key], bytes): + self.params[key] = self.params[key].decode(self.site.encoding()) if isinstance(self.params[key], basestring): # convert a stringified sequence into a list self.params[key] = self.params[key].split("|") @@ -192,7 +205,7 @@ self.params["meta"] = meta uiprop = self.params.get("uiprop", []) uiprop = set(uiprop + ["blockinfo", "hasmsg"]) - self.params["uiprop"] = list(uiprop) + self.params["uiprop"] = list(sorted(uiprop)) if "properties" in self.params: if "info" in self.params["properties"]: inprop = self.params.get("inprop", []) @@ -208,18 +221,20 @@ for key in self.params: try: self.params[key] = "|".join(self.params[key]) - if isinstance(self.params[key], unicode): - self.params[key] = self.params[key].encode(self.site.encoding()) + self.params[key] = self.params[key].encode(self.site.encoding()) except Exception: pywikibot.error( u"http_params: Key '%s' could not be encoded to '%s'; params=%r" % (key, self.site.encoding(), self.params[key])) - return urllib.urlencode(self.params) + return urlencode(self.params) def __str__(self): - return urllib.unquote(self.site.scriptpath() + return unquote(self.site.scriptpath() + "/api.php?" + self.http_params()) + + def __repr__(self): + return "%s.%s<%s->%r>" % (self.__class__.__module__, self.__class__.__name__, self.site, str(self)) def _simulate(self, action): if action and config.simulate and action in config.actions_to_block: @@ -441,8 +456,14 @@ # directory already exists pass + def _uniquedescriptionstr(self): + return (repr(self.site) + repr(sorted(self.iteritems()))) + def _create_file_name(self): - return hashlib.sha256(str(self.site) + str(self)).hexdigest() + self.http_params() # normalize self.iteritems() + return hashlib.sha256( + self._uniquedescriptionstr().encode('utf-8') + ).hexdigest() def _cachefile_path(self): return os.path.join(self._get_cache_dir(), self._create_file_name()) @@ -453,20 +474,25 @@ def _load_cache(self): """ Returns whether the cache can be used """ try: - sitestr, selfstr, self._data, self._cachetime = pickle.load(open(self._cachefile_path())) - assert(sitestr == str(self.site)) - assert(selfstr == str(self)) + with open(self._cachefile_path(), 'rb') as f: + uniquedescr, self._data, self._cachetime = pickle.load(f) + assert(uniquedescr == str(self._uniquedescriptionstr())) if self._expired(self._cachetime): self._data = None return False return True - except Exception: + except IOError as e: + # file not found + return False + except Exception as e: + pywikibot.output("Could not load cache: %r" % e) return False def _write_cache(self, data): """ writes data to self._cachefile_path() """ - data = [str(self.site), str(self), data, datetime.datetime.now()] - pickle.dump(data, open(self._cachefile_path(), 'w')) + data = [self._uniquedescriptionstr(), data, datetime.datetime.now()] + with open(self._cachefile_path(), 'wb') as f: + pickle.dump(data, f) def submit(self): cached_available = self._load_cache() @@ -661,7 +687,9 @@ new_limit = min(self.query_limit, self.limit - count) else: new_limit = None - if "rvprop" in self.request \ + + if new_limit and \ + "rvprop" in self.request \ and "content" in self.request["rvprop"]: # queries that retrieve page content have lower limits # Note: although API allows up to 500 pages for content @@ -727,7 +755,7 @@ # otherwise we proceed as usual else: count += 1 - if self.limit > 0 and count >= self.limit: + if self.limit and count >= self.limit: return if self.module == "random" and self.limit: # "random" module does not return "query-continue" diff --git a/tests/__init__.py b/tests/__init__.py index 92b921d..a9b675e 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -26,8 +26,12 @@ def patch_request(): + global original_expired pywikibot.data.api.Request = TestRequest + original_expired = pywikibot.data.api.CachedRequest._expired + pywikibot.data.api.CachedRequest._expired = lambda *args, **kwargs: False def unpatch_request(): pywikibot.data.api.Request = _original_Request + pywikibot.data.api.CachedRequest._expired = original_expired diff --git a/tests/site_tests.py b/tests/site_tests.py index bdee6d2..65fb17a 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -992,6 +992,7 @@ # Implemented without setUpClass(cls) and global variables as objects # were not completely disposed and recreated but retained 'memory' def setUp(self): + super(TestSiteLoadRevisions, self).setUp() code, family = "en", "wikipedia" self.mysite = pywikibot.Site(code, family) self.mainpage = pywikibot.Page(pywikibot.Link("Main Page", self.mysite)) -- To view, visit https://gerrit.wikimedia.org/r/130019 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ib3ff27f5f41f2236eb5510c96751936ac7e98f1e Gerrit-PatchSet: 4 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Merlijn van Deen &lt;valhallasw(a)arctus.nl&gt; Gerrit-Reviewer: Ladsgroup &lt;ladsgroup(a)gmail.com&gt; Gerrit-Reviewer: Merlijn van Deen &lt;valhallasw(a)arctus.nl&gt; Gerrit-Reviewer: Xqt &lt;info(a)gno.de&gt; Gerrit-Reviewer: jenkins-bot <>