http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10370
Revision: 10370 Author: valhallasw Date: 2012-06-17 19:13:57 +0000 (Sun, 17 Jun 2012) Log Message: ----------- Updated site_tests - requests are now cached (but the tests are still terribly slow) - removed module-level code that caused deadlocks (details below) - updated several site_tests, although still not all tests pass
Deadlock description: Expected behaviour:
``` Q = queue.Queue()
[pwb code] [threadedhttp] obj=Request() obj=Q.get() [blocks] | : Q.put(obj) ----------------------> : | | obj.lock.acquire() [blocks] do stuff with obj; import sys : | : <------------------------ obj.lock.release() | | V V ```
however, due to an import lock, this is what happens: (*) marks what happens in cpython internally
``` Q = queue.Queue() import_lock = Lock() (*)
[pwb code] [threadedhttp] import_lock.acquire() (*) obj=Q.get() [blocks] | : obj=Request(); : | : Q.put(obj); ----------------------> : | | obj.lock.acquire() [blocks] import sys [causing, internally: : import_lock.acquire() [blocks] (*) : : === (deadlock) === ```
The import is caused by nosetests, which loads the code to run the tests.
This is fixed by moving the requests to inside the test class instead of having it in module level code.
Modified Paths: -------------- branches/rewrite/tests/__init__.py branches/rewrite/tests/site_tests.py
Modified: branches/rewrite/tests/__init__.py =================================================================== --- branches/rewrite/tests/__init__.py 2012-06-17 18:03:29 UTC (rev 10369) +++ branches/rewrite/tests/__init__.py 2012-06-17 19:13:57 UTC (rev 10370) @@ -5,3 +5,33 @@ # Distributed under the terms of the MIT license. # __version__ = '$Id$' + +import os +import pywikibot.data.api +from pywikibot.data.api import Request as _original_Request +from pywikibot.data.api import CachedRequest + +class TestRequest(CachedRequest): + def __init__(self, *args, **kwargs): + super(TestRequest, self).__init__(0, *args, **kwargs) + + def _get_cache_dir(self): + path = os.path.join(os.path.split(__file__)[0], 'apicache') + self._make_dir(path) + return path + + def _expired(self, dt): + return False + + def submit(self): + cached_available = self._load_cache() + if not cached_available: + print str(self) + return super(TestRequest, self).submit() + +def patch_request(): + pywikibot.data.api.Request = TestRequest + +def unpatch_request(): + pywikibot.data.api.Request = _original_Request +
Modified: branches/rewrite/tests/site_tests.py =================================================================== --- branches/rewrite/tests/site_tests.py 2012-06-17 18:03:29 UTC (rev 10369) +++ branches/rewrite/tests/site_tests.py 2012-06-17 19:13:57 UTC (rev 10370) @@ -13,14 +13,27 @@ import unittest import pywikibot import warnings +from tests import patch_request, unpatch_request
-mysite = pywikibot.Site() -mainpage = pywikibot.Page(pywikibot.Link("Main Page", mysite)) -imagepage = iter(mainpage.imagelinks()).next() # 1st image on main page +mysite = None +mainpage = None +imagepage = None
- class TestSiteObject(unittest.TestCase): """Test cases for Site methods.""" + family = "wikipedia" + code = "en" + @classmethod + def setUpClass(cls): + patch_request() + global mysite, mainpage, imagepage + mysite = pywikibot.Site(cls.code, cls.family) + mainpage = pywikibot.Page(pywikibot.Link("Main Page", mysite)) + imagepage = iter(mainpage.imagelinks()).next() # 1st image on main page + + @classmethod + def tearDownClass(cls): + unpatch_request
def assertType(self, obj, cls): """Assert that obj is an instance of type cls""" @@ -29,17 +42,17 @@ def testBaseMethods(self): """Test cases for BaseSite methods"""
- self.assertEqual(mysite.family.name, pywikibot.config2.family) - self.assertEqual(mysite.code, pywikibot.config2.mylang) + self.assertEqual(mysite.family.name, self.family) + self.assertEqual(mysite.code, self.code) self.assertType(mysite.lang, basestring) self.assertType(mysite == pywikibot.Site("en", "wikipedia"), bool) self.assertType(mysite.user(), (basestring, type(None))) self.assertEqual(mysite.sitename(), - "%s:%s" % (pywikibot.config2.family, - pywikibot.config2.mylang)) + "%s:%s" % (self.family, + self.code)) self.assertEqual(repr(mysite), 'Site("%s", "%s")' - % (pywikibot.config2.mylang, pywikibot.config2.family)) + % (self.code, self.family)) self.assertType(mysite.linktrail(), basestring) self.assertType(mysite.redirect(default=True), basestring) self.assertType(mysite.disambcategory(), pywikibot.Category) @@ -62,10 +75,9 @@ obs = mysite.family.obsolete ipf = mysite.interwiki_putfirst() self.assertType(ipf, list) - self.assertTrue(all(item in langs or item in obs - for item in ipf)) - self.assertTrue(all(item in langs - for item in mysite.validLanguageLinks())) + + for item in mysite.validLanguageLinks(): + self.assertTrue(item in langs, item)
def testNamespaceMethods(self): """Test cases for methods manipulating namespace names""" @@ -192,7 +204,8 @@ filterRedirects=True)) # including links to redirect pages (but not the redirects): indirect = set(mysite.pagebacklinks(mainpage, namespaces=[0], - followRedirects=True)) + followRedirects=True, + filterRedirects=False)) self.assertEqual(filtered & redirs, set([])) self.assertEqual(indirect & redirs, set([])) self.assertTrue(filtered.issubset(indirect))