Pywikibot-commits September 2014

pywikibot-commits@lists.wikimedia.org

1 participants
178 discussions

[Gerrit] [FEAT] Handle API warnings - change (pywikibot/core)
by jenkins-bot (Code Review) 07 Sep '14

07 Sep '14

jenkins-bot has submitted this change and it was merged. Change subject: [FEAT] Handle API warnings ...................................................................... [FEAT] Handle API warnings Some code parses the warnings returned by the API separately like Siteinfo. This option adds the possibility to handle warnings returned by the api via "_warning_handler". If it's not a method or that method returns not True the warning is still output like before (and is by default). The method must accept two parameters and is executed for each warning. This also shows warnings when CachedRequest are used and not only for the first actually submitted request. Change-Id: I62d13dca5b592bddf72dbbdcc0b8edf77c0059ed --- M pywikibot/data/api.py M pywikibot/site.py 2 files changed, 44 insertions(+), 24 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index c36f88e..56c42c0 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -160,6 +160,7 @@ if "action" not in kwargs: raise ValueError("'action' specification missing from Request.") self.update(**kwargs) + self._warning_handler = None # Actions that imply database updates on the server, used for various # things like throttling or skipping actions when we're in simulation # mode @@ -342,6 +343,25 @@ submsg.set_payload(content) return submsg + def _handle_warnings(self, result): + if 'warnings' in result: + for mod, warning in result['warnings'].items(): + if mod == 'info': + continue + if '*' in warning: + text = warning['*'] + elif 'html' in warning: + # Bugzilla 49978 + text = warning['html']['*'] + else: + pywikibot.warning( + u'API warning ({0})of unknown format: {1}'. + format(mod, warning)) + continue + if (not callable(self._warning_handler) or + not self._warning_handler(mod, text)): + pywikibot.warning(u"API warning (%s): %s" % (mod, text)) + def submit(self): """Submit a query and parse the response. @@ -470,21 +490,7 @@ self.site.login(status) # retry the previous query continue - if "warnings" in result: - modules = [k for k in result["warnings"] if k != "info"] - for mod in modules: - if '*' in result["warnings"][mod]: - text = result["warnings"][mod]['*'] - elif 'html' in result["warnings"][mod]: - # Bugzilla 49978 - text = result["warnings"][mod]['html']['*'] - else: - # This is just a warning, we shouldn't raise an - # exception because of it - continue - pywikibot.warning( - u"API warning (%s): %s" - % (mod, text)) + self._handle_warnings(result) if "error" not in result: return result if "*" in result["error"]: @@ -639,6 +645,8 @@ if not cached_available: self._data = super(CachedRequest, self).submit() self._write_cache(self._data) + else: + self._handle_warnings(self._data) return self._data diff --git a/pywikibot/site.py b/pywikibot/site.py index 46995af..4b4848d 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -907,6 +907,9 @@ All values of the siteinfo property 'general' are directly available. """ + WARNING_REGEX = re.compile(u"^Unrecognized values? for parameter " + u"'siprop': ([^,]+(?:, [^,]+)*)$") + def __init__(self, site): """Initialise it with an empty cache.""" self._site = site @@ -957,19 +960,33 @@ @rtype: dict (the values) @see: U{https://www.mediawiki.org/wiki/API:Meta#siteinfo_.2F_si} """ + def warn_handler(mod, message): + """Return True if the warning is handled.""" + matched = Siteinfo.WARNING_REGEX.match(message) + if mod == 'siteinfo' and matched: + invalid_properties.extend( + prop.strip() for prop in matched.group(1).split(',')) + return True + else: + return False + if isinstance(prop, basestring): props = [prop] else: props = prop if len(props) == 0: raise ValueError('At least one property name must be provided.') + invalid_properties = [] try: - data = pywikibot.data.api.CachedRequest( + request = pywikibot.data.api.CachedRequest( expiry=pywikibot.config.API_config_expiry if expiry is False else expiry, site=self._site, action='query', meta='siteinfo', - siprop='|'.join(props)).submit() + siprop='|'.join(props)) + # warnings are handled later + request._warning_handler = warn_handler + data = request.submit() except api.APIError as e: if e.code == 'siunknown_siprop': if len(props) == 1: @@ -987,13 +1004,8 @@ raise else: result = {} - if 'warnings' in data and 'siteinfo' in data['warnings']: - invalid_properties = [] - for prop in re.match(u"^Unrecognized values? for parameter " - u"'siprop': ([^,]+(?:, [^,]+)*)$", - data['warnings']['siteinfo']['*']).group(1).split(','): - prop = prop.strip() - invalid_properties += [prop] + if invalid_properties: + for prop in invalid_properties: result[prop] = (Siteinfo._get_default(prop), False) pywikibot.log(u"Unable to get siprop(s) '{0}'".format( u"', '".join(invalid_properties))) -- To view, visit https://gerrit.wikimedia.org/r/155235 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I62d13dca5b592bddf72dbbdcc0b8edf77c0059ed Gerrit-PatchSet: 6 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] Require login for two site methods - change (pywikibot/core)
by jenkins-bot (Code Review) 07 Sep '14

07 Sep '14

jenkins-bot has submitted this change and it was merged. Change subject: Require login for two site methods ...................................................................... Require login for two site methods Also purgepages was recently added as a API 'write' action which asserts it must be performed by a logged in user. is_uploaddisabled added recently for bug 69090 only works if the user is logged in, otherwise the assert module returns an error before the upload module can respond. Re-raise any API error that is_uploaddisabled is not expecting. Use test projects to test is_uploaddisabled, as the travis config already has username's for those sites. Change-Id: I009260c9476852cc8ee818aa791fd15f1dc05ec1 --- M pywikibot/site.py M tests/site_tests.py 2 files changed, 11 insertions(+), 3 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/site.py b/pywikibot/site.py index 6825a26..bcead58 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -3885,6 +3885,7 @@ return ((unwatch and "unwatched" in watched) or (not unwatch and "watched" in result)) + @must_be(group='user') def purgepages(self, pages, **kwargs): """Purge the server's cache for one or multiple pages. @@ -3936,6 +3937,7 @@ def getImagesFromAnHash(self, hash_found=None): return self.getFilesFromAnHash(hash_found) + @must_be(group='user') def is_uploaddisabled(self): """Return True if upload is disabled on site. @@ -3959,8 +3961,14 @@ except api.APIError as error: if error.code == u'uploaddisabled': self._uploaddisabled = True - else: + elif error.code == u'missingparam': + # If the upload module is enabled, the above dummy request + # does not have sufficient parameters and will cause a + # 'missingparam' error. self._uploaddisabled = False + else: + # Unexpected error + raise return self._uploaddisabled @deprecate_arg('imagepage', 'filepage') diff --git a/tests/site_tests.py b/tests/site_tests.py index 5f72381..f3d8899 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -1184,10 +1184,10 @@ class TestUploadEnabledSite(SiteTestCase): def test_is_uploaddisabled(self): - site = pywikibot.Site('commons', 'commons') + site = pywikibot.Site('test', 'wikipedia') self.assertFalse(site.is_uploaddisabled()) - site = pywikibot.Site('wikidata', 'wikidata') + site = pywikibot.Site('test', 'wikidata') self.assertTrue(site.is_uploaddisabled()) -- To view, visit https://gerrit.wikimedia.org/r/158918 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I009260c9476852cc8ee818aa791fd15f1dc05ec1 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] site.py: check if site is enabled for uploads - change (pywikibot/core)
by jenkins-bot (Code Review) 07 Sep '14

07 Sep '14

jenkins-bot has submitted this change and it was merged. Change subject: site.py: check if site is enabled for uploads ...................................................................... site.py: check if site is enabled for uploads Fix Bug 69090: upload.py attempts uploads on wikis that don't allow uploads. Check APIError code to determine if Site is enabled for uploading. Currently it is done and cached at first upload attempt. Added also a helper function is_uploaddisabled() to determine if a site has uploads enabled, based on a dummy upload attempt. Throttle has been disabled for such request to speed up the query (implementation taken from I80b2bba9e63832173d5b697db1f4ea419ca1122f). Change-Id: Id8ae0cd51582d76b7bdd15c53e39b4105995b540 --- M pywikibot/site.py M scripts/upload.py M tests/site_tests.py 3 files changed, 60 insertions(+), 7 deletions(-) Approvals: John Vandenberg: Looks good to me, approved XZise: Looks good to me, but someone else must approve jenkins-bot: Verified diff --git a/pywikibot/site.py b/pywikibot/site.py index cfbd128..214f638 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -3897,6 +3897,33 @@ def getImagesFromAnHash(self, hash_found=None): return self.getFilesFromAnHash(hash_found) + def is_uploaddisabled(self): + """Return True if upload is disabled on site. + + If not called directly, it is cached by the first attempted + upload action. + + """ + if hasattr(self, '_uploaddisabled'): + return self._uploaddisabled + else: + # attempt a fake upload; on enabled sites will fail for: + # missingparam: One of the parameters + # filekey, file, url, statuskey is required + # TODO: is there another way? + try: + dummy = pywikibot.FilePage(self, title='dummy') + token = self.token(dummy, 'edit') + req = api.Request(site=self, action="upload", + token=token, throttle=False) + req.submit() + except api.APIError as error: + if error.code == u'uploaddisabled': + self._uploaddisabled = True + else: + self._uploaddisabled = False + return self._uploaddisabled + @deprecate_arg('imagepage', 'filepage') def upload(self, filepage, source_filename=None, source_url=None, comment=None, text=None, watch=False, ignore_warnings=False, @@ -3923,6 +3950,7 @@ and the chunk size is positive but lower than the file size. @type chunk_size: int """ + upload_warnings = { # map API warning codes to user error messages # %(msg)s will be replaced by message string from API responsse @@ -3980,8 +4008,14 @@ req.mime_params['chunk'] = (chunk, None, {'filename': req.params['filename']}) if file_key: req['filekey'] = file_key - # TODO: Proper error and warning handling - data = req.submit()['upload'] + try: + data = req.submit()['upload'] + self._uploaddisabled = False + except api.APIError as error: + # TODO: catch and process foreseeable errors + if error.code == u'uploaddisabled': + self._uploaddisabled = True + raise error if 'warnings' in data: result = data break @@ -4016,11 +4050,15 @@ req["ignorewarnings"] = "" try: result = req.submit() - except api.APIError: + self._uploaddisabled = False + except api.APIError as error: # TODO: catch and process foreseeable errors - raise + if error.code == u'uploaddisabled': + self._uploaddisabled = True + raise error result = result["upload"] - pywikibot.debug(result, _logger) + pywikibot.debug(result, _logger) + if "warnings" in result: warning = list(result["warnings"].keys())[0] message = result["warnings"][warning] diff --git a/scripts/upload.py b/scripts/upload.py index 5ab1607..2b6436e 100755 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -270,7 +270,12 @@ else: pywikibot.output(u"Upload aborted.") return - + except pywikibot.data.api.APIError as error: + if error.code == u'uploaddisabled': + pywikibot.error("Upload error: Local file uploads are disabled on %s." + % site) + else: + pywikibot.error("Upload error: ", exc_info=True) except Exception: pywikibot.error("Upload error: ", exc_info=True) diff --git a/tests/site_tests.py b/tests/site_tests.py index 125b6aa..df636c2 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -16,7 +16,7 @@ import pywikibot from pywikibot.exceptions import Error, NoPage -from tests.utils import PywikibotTestCase, unittest +from tests.utils import SiteTestCase, PywikibotTestCase, unittest if sys.version_info[0] > 2: basestring = (str, ) @@ -1179,6 +1179,16 @@ self.assertEqual(ll.site.family.name, 'wikipedia') +class TestUploadEnabledSite(SiteTestCase): + + def test_is_uploaddisabled(self): + site = pywikibot.Site('commons', 'commons') + self.assertFalse(site.is_uploaddisabled()) + + site = pywikibot.Site('wikidata', 'wikidata') + self.assertTrue(site.is_uploaddisabled()) + + if __name__ == '__main__': try: unittest.main() -- To view, visit https://gerrit.wikimedia.org/r/156465 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Id8ae0cd51582d76b7bdd15c53e39b4105995b540 Gerrit-PatchSet: 6 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] Determine entity namespaces for site using API - change (pywikibot/core)
by XZise (Code Review) 07 Sep '14

07 Sep '14

XZise has submitted this change and it was merged. Change subject: Determine entity namespaces for site using API ...................................................................... Determine entity namespaces for site using API Remove hardcoded namespace numbers for 0 and 120 replacing them with values obtained via existing siprop=namespaces query by new Namespace class. Move namespace=0 title reparsing into ItemPage. Description of class PropertyPage should not recommend including namespace in the title as namespace is and was already provided. Change-Id: If7ca06adbb4b9932bba0abffc7588afcb320e934 --- M pywikibot/exceptions.py M pywikibot/page.py M pywikibot/site.py M tests/wikibase_tests.py 4 files changed, 340 insertions(+), 24 deletions(-) Approvals: XZise: Looks good to me, approved diff --git a/pywikibot/exceptions.py b/pywikibot/exceptions.py index 661fcf5..7feba4a 100644 --- a/pywikibot/exceptions.py +++ b/pywikibot/exceptions.py @@ -37,6 +37,7 @@ WikiBaseError: any issue specific to Wikibase. - CoordinateGlobeUnknownException: globe is not implemented yet. + - EntityTypeUnknownException: entity type is not available on the site. """ # @@ -297,6 +298,15 @@ """ This globe is not implemented yet in either WikiBase or pywikibot """ + pass + + +class EntityTypeUnknownException(WikiBaseError): + + """The requested entity type is not recognised on this site""" + + pass + # TODO: Warn about the deprecated usage import pywikibot.data.api UploadWarning = pywikibot.data.api.UploadWarning diff --git a/pywikibot/page.py b/pywikibot/page.py index 18c6c9c..1fc54eb 100644 --- a/pywikibot/page.py +++ b/pywikibot/page.py @@ -20,7 +20,7 @@ import sys import pywikibot from pywikibot import config -import pywikibot.site +from pywikibot.site import Namespace from pywikibot.exceptions import AutoblockUser, UserActionRefuse from pywikibot.tools import ComparableMixin, deprecated, deprecate_arg from pywikibot import textlib @@ -2612,24 +2612,96 @@ """ def __init__(self, site, title=u"", **kwargs): - """ Constructor. """ + """ Constructor. + + If title is provided, either ns or entity_type must also be provided, + and will be checked against the title parsed using the Page + initialisation logic. + + @param site: Wikibase data site + @type site: DataSite + @param title: normalized title of the page + @type title: unicode + @param ns: namespace + @type ns: Namespace instance, or int + @param entity_type: Wikibase entity type + @type entity_type: str ('item' or 'property') + + @raise TypeError: incorrect use of parameters + @raise ValueError: incorrect namespace + @raise pywikibot.Error: title parsing problems + @raise NotImplementedError: the entity type is not supported + """ if not isinstance(site, pywikibot.site.DataSite): raise TypeError("site must be a pywikibot.site.DataSite object") + if title and ('ns' not in kwargs and 'entity_type' not in kwargs): + pywikibot.debug("%s.__init__: %s title %r specified without " + "ns or entity_type" + % (self.__class__.__name__, site, title), + layer='wikibase') + + self._namespace = None + + if 'ns' in kwargs: + if isinstance(kwargs['ns'], Namespace): + self._namespace = kwargs.pop('ns') + kwargs['ns'] = self._namespace.id + else: + # numerical namespace given + ns = int(kwargs['ns']) + if site.item_namespace.id == ns: + self._namespace = site.item_namespace + elif site.property_namespace.id == ns: + self._namespace = site.property_namespace + else: + raise ValueError('%r: Namespace "%d" is not valid' + % self.site) + + if 'entity_type' in kwargs: + entity_type = kwargs.pop('entity_type') + if entity_type == 'item': + entity_type_ns = site.item_namespace + elif entity_type == 'property': + entity_type_ns = site.property_namespace + else: + raise ValueError('Wikibase entity type "%s" unknown' + % entity_type) + + if self._namespace: + if self._namespace != entity_type_ns: + raise ValueError('Namespace "%d" is not valid for Wikibase' + ' entity type "%s"' + % (kwargs['ns'], entity_type)) + else: + self._namespace = entity_type_ns + kwargs['ns'] = self._namespace.id + Page.__init__(self, site, title, **kwargs) + + # If a title was not provided, + # avoid checks which may cause an exception. + if not title: + self.repo = site + return + + if self._namespace: + if self._link.namespace != self._namespace.id: + raise ValueError(u"'%s' is not in the namespace %d" + % (title, self._namespace.id)) + else: + # Neither ns or entity_type was provided. + # Use the _link to determine entity type. + ns = self._link.namespace + if self.site.item_namespace.id == ns: + self._namespace = self.site.item_namespace + elif self.site.property_namespace.id == ns: + self._namespace = self.site.property_namespace + else: + raise ValueError('%r: Namespace "%d" is not valid' + % (self.site, ns)) + + # .site forces a parse of the Link title to determine site self.repo = self.site - - def title(self, **kwargs): - """ Page title. - - If the item was instantiated without an ID, - fetch the ID and reparse the title. - """ - if self.namespace() == 0: - self.getID() - if self._link._text != self.id: - self._link._text = self.id - del self._link._title - return Page(self).title(**kwargs) def _defined_by(self, singular=False): """ @@ -2665,6 +2737,14 @@ params[id] = self.getID() return params + + def namespace(self): + """Return the number of the namespace of the entity. + + @return: Namespace id + @rtype: int + """ + return self._namespace.id def exists(self): """ @@ -2872,7 +2952,7 @@ class ItemPage(WikibasePage): - """ A Wikibase item. + """ Wikibase entity of type 'item'. A Wikibase item may be defined by either a 'Q' id (qid), or by a site & title. @@ -2888,9 +2968,26 @@ @param site: data repository @type site: pywikibot.site.DataSite @param title: id number of item, "Q###" + @type title: str """ - super(ItemPage, self).__init__(site, title, ns=0) - self.id = title.upper() # This might cause issues if not ns0? + super(ItemPage, self).__init__(site, title, + ns=site.item_namespace) + self.id = self._link.title.upper() + + def title(self, **kwargs): + """ + Get the title of the page. + + All optional keyword parameters are passed to the superclass. + """ + # If the item was instantiated without an ID, + # remove the existing Link title, force the Link text to be reparsed. + self.getID() + if self._link._text != self.id: + self._link._text = self.id + del self._link._title + + return super(ItemPage, self).title(**kwargs) @classmethod def fromPage(cls, page): @@ -3107,6 +3204,8 @@ """ Constructor. + @param site: data repository + @type site: pywikibot.site.DataSite @param datatype: datatype of the property; if not given, it will be queried via the API @type datatype: basestring @@ -3160,7 +3259,7 @@ A Wikibase entity in the property namespace. Should be created as: - PropertyPage(DataSite, 'Property:P21') + PropertyPage(DataSite, 'P21') """ def __init__(self, source, title=u""): @@ -3169,9 +3268,11 @@ @param source: data repository property is on @type source: pywikibot.site.DataSite - @param title: page name of property, like "Property:P##" + @param title: page name of property, like "P##" + @type title: str """ - WikibasePage.__init__(self, source, title, ns=120) + WikibasePage.__init__(self, source, title, + ns=source.property_namespace) Property.__init__(self, source, title) self.id = self.title(withNamespace=False).upper() if not self.id.startswith(u'P'): diff --git a/pywikibot/site.py b/pywikibot/site.py index 2389346..39d8e5f 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -4334,6 +4334,62 @@ class DataSite(APISite): + """Wikibase data capable site.""" + + def __init__(self, code, fam, user, sysop): + """Constructor.""" + APISite.__init__(self, code, fam, user, sysop) + self._item_namespace = None + self._property_namespace = None + + def _cache_entity_namespaces(self): + """Find namespaces for each known wikibase entity type.""" + self._item_namespace = False + self._property_namespace = False + + for namespace in self.namespaces().values(): + content_model = namespace.info.get('defaultcontentmodel') + if content_model == 'wikibase-item': + self._item_namespace = namespace + elif content_model == 'wikibase-property': + self._property_namespace = namespace + + @property + def item_namespace(self): + """ + Return namespace for items. + + @return: item namespace + @rtype: Namespace + """ + if self._item_namespace is None: + self._cache_entity_namespaces() + + if isinstance(self._item_namespace, Namespace): + return self._item_namespace + else: + raise pywikibot.exceptions.EntityTypeUnknownException( + '%r does not support entity type "item"' + % self) + + @property + def property_namespace(self): + """ + Return namespace for properties. + + @return: property namespace + @rtype: Namespace + """ + if self._property_namespace is None: + self._cache_entity_namespaces() + + if isinstance(self._property_namespace, Namespace): + return self._property_namespace + else: + raise pywikibot.exceptions.EntityTypeUnknownException( + '%r does not support entity type "property"' + % self) + def __getattr__(self, attr): """Provide data access methods. diff --git a/tests/wikibase_tests.py b/tests/wikibase_tests.py index 9e1bca9..8d3698b 100644 --- a/tests/wikibase_tests.py +++ b/tests/wikibase_tests.py @@ -12,6 +12,7 @@ import pywikibot from pywikibot import pagegenerators from pywikibot.page import WikibasePage +from pywikibot.site import Namespace from pywikibot.data.api import APIError import json import copy @@ -238,11 +239,10 @@ check(title, APIError) def test_item_untrimmed_title(self): - # spaces in the title cause an error item = pywikibot.ItemPage(wikidata, ' Q60 ') self.assertEqual(item._link._title, 'Q60') - self.assertEqual(item.title(), ' Q60 ') - self.assertRaises(APIError, item.get) + self.assertEqual(item.title(), 'Q60') + item.get() def test_item_missing(self): # this item is deleted @@ -390,9 +390,11 @@ class TestPropertyPage(PywikibotTestCase): def test_property_empty_property(self): + """Test creating a PropertyPage without a title.""" self.assertRaises(pywikibot.Error, pywikibot.PropertyPage, wikidata) def test_globe_coordinate(self): + """Test a coordinate PropertyPage has the correct type.""" property_page = pywikibot.PropertyPage(wikidata, 'P625') self.assertEqual(property_page.type, 'globe-coordinate') self.assertEqual(property_page.getType(), 'globecoordinate') @@ -525,6 +527,153 @@ self.assertEqual(response, self.data_out) +class TestNamespaces(PywikibotTestCase): + """Test cases to test namespaces of Wikibase entities""" + + def test_empty_wikibase_page(self): + # As a base class it should be able to instantiate + # it with minimal arguments + page = pywikibot.page.WikibasePage(wikidata) + self.assertRaises(AttributeError, page.namespace) + page = pywikibot.page.WikibasePage(wikidata, title='') + self.assertRaises(AttributeError, page.namespace) + + page = pywikibot.page.WikibasePage(wikidata, ns=0) + self.assertEqual(page.namespace(), 0) + page = pywikibot.page.WikibasePage(wikidata, entity_type='item') + self.assertEqual(page.namespace(), 0) + + page = pywikibot.page.WikibasePage(wikidata, ns=120) + self.assertEqual(page.namespace(), 120) + page = pywikibot.page.WikibasePage(wikidata, title='', ns=120) + self.assertEqual(page.namespace(), 120) + page = pywikibot.page.WikibasePage(wikidata, entity_type='property') + self.assertEqual(page.namespace(), 120) + + # mismatch in namespaces + self.assertRaises(ValueError, pywikibot.page.WikibasePage, wikidata, + ns=0, entity_type='property') + self.assertRaises(ValueError, pywikibot.page.WikibasePage, wikidata, + ns=120, entity_type='item') + + def test_wikibase_link_namespace(self): + """Test the title resolved to a namespace correctly.""" + # title without any namespace clues (ns or entity_type) + # should verify the Link namespace is appropriate + page = pywikibot.page.WikibasePage(wikidata, title='Q6') + self.assertEqual(page.namespace(), 0) + page = pywikibot.page.WikibasePage(wikidata, title='Property:P60') + self.assertEqual(page.namespace(), 120) + + def test_wikibase_namespace_selection(self): + """Test various ways to correctly specify the namespace.""" + page = pywikibot.page.ItemPage(wikidata, 'Q6') + self.assertEqual(page.namespace(), 0) + page = pywikibot.page.ItemPage(wikidata, title='Q6') + self.assertEqual(page.namespace(), 0) + + page = pywikibot.page.WikibasePage(wikidata, title='Q6', ns=0) + self.assertEqual(page.namespace(), 0) + page = pywikibot.page.WikibasePage(wikidata, title='Q6', + entity_type='item') + self.assertEqual(page.namespace(), 0) + + page = pywikibot.page.PropertyPage(wikidata, 'Property:P60') + self.assertEqual(page.namespace(), 120) + page = pywikibot.page.PropertyPage(wikidata, 'P60') + self.assertEqual(page.namespace(), 120) + + page = pywikibot.page.WikibasePage(wikidata, title='P60', ns=120) + self.assertEqual(page.namespace(), 120) + page = pywikibot.page.WikibasePage(wikidata, title='P60', + entity_type='property') + self.assertEqual(page.namespace(), 120) + + def test_wrong_namespaces(self): + """Test incorrect namespaces for Wikibase entities.""" + # All subclasses of WikibasePage raise a ValueError + # if the namespace for the page title is not correct + self.assertRaises(ValueError, pywikibot.page.WikibasePage, wikidata, + title='Wikidata:Main Page') + self.assertRaises(ValueError, pywikibot.ItemPage, wikidata, 'File:Q1') + self.assertRaises(ValueError, pywikibot.PropertyPage, wikidata, 'File:P60') + + def test_item_unknown_namespace(self): + """Test unknown namespaces for Wikibase entities.""" + # The 'Invalid:' is not a known namespace, so is parsed to be + # part of the title in namespace 0 + # TODO: These items have inappropriate titles, which should + # raise an error. + item = pywikibot.ItemPage(wikidata, 'Invalid:Q1') + self.assertEqual(item.namespace(), 0) + self.assertEqual(item.id, 'INVALID:Q1') + self.assertEqual(item.title(), 'INVALID:Q1') + self.assertEqual(hasattr(item, '_content'), False) + self.assertRaises(APIError, item.get) + self.assertEqual(hasattr(item, '_content'), False) + self.assertEqual(item.title(), 'INVALID:Q1') + + +class TestAlternateNamespaces(PywikibotTestCase): + """Test cases to test namespaces of Wikibase entities""" + + def setUp(self): + super(TestAlternateNamespaces, self).setUp() + + class DrySite(pywikibot.site.DataSite): + _namespaces = { + 90: Namespace(id=90, + canonical_name='Item', + defaultcontentmodel='wikibase-item'), + 92: Namespace(id=92, + canonical_name='Prop', + defaultcontentmodel='wikibase-property') + } + + __init__ = lambda *args: None + code = 'test' + family = lambda: None + family.name = 'test' + _logged_in_as = None + _siteinfo = {'case': 'first-letter'} + _item_namespace = None + _property_namespace = None + + def encoding(self): + return 'utf-8' + + def encodings(self): + return [] + + self.site = DrySite('test', 'mock', None, None) + + def test_alternate_item_namespace(self): + item = pywikibot.ItemPage(self.site, 'Q60') + self.assertEqual(item.namespace(), 90) + self.assertEqual(item.id, 'Q60') + self.assertEqual(item.title(), 'Item:Q60') + self.assertEqual(item._defined_by(), {'ids': 'Q60'}) + + item = pywikibot.ItemPage(self.site, 'Item:Q60') + self.assertEqual(item.namespace(), 90) + self.assertEqual(item.id, 'Q60') + self.assertEqual(item.title(), 'Item:Q60') + self.assertEqual(item._defined_by(), {'ids': 'Q60'}) + + def test_alternate_property_namespace(self): + prop = pywikibot.PropertyPage(self.site, 'P21') + self.assertEqual(prop.namespace(), 92) + self.assertEqual(prop.id, 'P21') + self.assertEqual(prop.title(), 'Prop:P21') + self.assertEqual(prop._defined_by(), {'ids': 'P21'}) + + prop = pywikibot.PropertyPage(self.site, 'Prop:P21') + self.assertEqual(prop.namespace(), 92) + self.assertEqual(prop.id, 'P21') + self.assertEqual(prop.title(), 'Prop:P21') + self.assertEqual(prop._defined_by(), {'ids': 'P21'}) + + if __name__ == '__main__': try: unittest.main() -- To view, visit https://gerrit.wikimedia.org/r/135385 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: If7ca06adbb4b9932bba0abffc7588afcb320e934 Gerrit-PatchSet: 22 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: JAn Dudík <jan.dudik(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Legoktm <legoktm.wikipedia(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: Ricordisamoa <ricordisamoa(a)openmailbox.org> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] [FIX] Travis: Add account for test.wiki(data/pedia) - change (pywikibot/core)
by jenkins-bot (Code Review) 06 Sep '14

06 Sep '14

jenkins-bot has submitted this change and it was merged. Change subject: [FIX] Travis: Add account for test.wiki(data/pedia) ...................................................................... [FIX] Travis: Add account for test.wiki(data/pedia) This account is required to do tests on wikidata introduced in c07c8da00cef8e189762ccf4f46422f6bb6f5d5f borrowed from I1e1ce836f35acb270c78331f1ec9c57f8ce4c0e2. Change-Id: Ief9c3e60e4e31e38ef149c97e8a72c55fc5eb273 --- M .travis.yml 1 file changed, 2 insertions(+), 0 deletions(-) Approvals: Merlijn van Deen: Looks good to me, approved jenkins-bot: Verified diff --git a/.travis.yml b/.travis.yml index b54adcc..96983cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,6 +20,8 @@ - echo "family = '$FAMILY'" >> ~/.pywikibot/user-config.py - echo "usernames['$FAMILY']['$LANGUAGE'] = 'Pywikibot-test'" >> ~/.pywikibot/user-config.py - echo "usernames['wikipedia']['en'] = 'Pywikibot-test'" >> ~/.pywikibot/user-config.py + - echo "usernames['wikipedia']['test'] = 'Pywikibot-test'" >> ~/.pywikibot/user-config.py + - echo "usernames['wikidata']['test'] = 'Pywikibot-test'" >> ~/.pywikibot/user-config.py - echo "password_file = os.path.expanduser('~/.pywikibot/passwordfile')" >> ~/.pywikibot/user-config.py - touch ~/.pywikibot/passwordfile -- To view, visit https://gerrit.wikimedia.org/r/158878 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ief9c3e60e4e31e38ef149c97e8a72c55fc5eb273 Gerrit-PatchSet: 3 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] Page save related exception hierarchy - change (pywikibot/core)
by XZise (Code Review) 06 Sep '14

06 Sep '14

XZise has submitted this change and it was merged. Change subject: Page save related exception hierarchy ...................................................................... Page save related exception hierarchy All page save related exceptions are now subclasses of the new PageSaveRelatedError, and PageNotSaved now is an alias of PageSaveRelatedError. Exceptions have been added for the known API error codes encountered by Site.editpage that are errors specific to the page being edited, and all page related save exceptions are re-raised to be handled by scripts. Scripts may continue to catch specific exceptions to customise actions or errors, however scripts now only need to catch one exception (PageNotSaved) to avoid halting on issues relating to a specific page. PageNotSaved has been moved to last in the exception handling of four scripts when it was previously before another exception which is now a subclass of PageNotSaved. Added edit_failure_tests to check exceptions are raised. Bug: 55264 Bug: 67276 Change-Id: Id8b8c8055100329e7844b5db6500f71bbb1673f7 --- M pywikibot/__init__.py M pywikibot/exceptions.py M pywikibot/page.py M pywikibot/site.py M scripts/add_text.py M scripts/blockpageschecker.py M scripts/reflinks.py M scripts/replace.py A tests/edit_failure_tests.py 9 files changed, 236 insertions(+), 68 deletions(-) Approvals: XZise: Looks good to me, approved diff --git a/pywikibot/__init__.py b/pywikibot/__init__.py index 0228e79..184d875 100644 --- a/pywikibot/__init__.py +++ b/pywikibot/__init__.py @@ -37,7 +37,9 @@ Error, InvalidTitle, BadTitle, NoPage, SectionError, NoSuchSite, NoUsername, UserBlocked, PageRelatedError, IsRedirectPage, IsNotRedirectPage, - PageNotSaved, LockedPage, EditConflict, + PageSaveRelatedError, PageNotSaved, OtherPageSaveError, + LockedPage, CascadeLockedPage, LockedNoPage, + EditConflict, PageDeletedConflict, PageCreatedConflict, ServerError, FatalServerError, Server504Error, CaptchaError, SpamfilterError, CircularRedirect, WikiBaseError, CoordinateGlobeUnknownException, @@ -69,7 +71,10 @@ 'Error', 'InvalidTitle', 'BadTitle', 'NoPage', 'SectionError', 'NoSuchSite', 'NoUsername', 'UserBlocked', 'PageRelatedError', 'IsRedirectPage', 'IsNotRedirectPage', - 'PageNotSaved', 'UploadWarning', 'LockedPage', 'EditConflict', + 'PageSaveRelatedError', 'PageNotSaved', 'OtherPageSaveError', + 'LockedPage', 'CascadeLockedPage', 'LockedNoPage', + 'EditConflict', 'PageDeletedConflict', 'PageCreatedConflict', + 'UploadWarning', 'ServerError', 'FatalServerError', 'Server504Error', 'CaptchaError', 'SpamfilterError', 'CircularRedirect', 'WikiBaseError', 'CoordinateGlobeUnknownException', diff --git a/pywikibot/exceptions.py b/pywikibot/exceptions.py index 35f7d2d..661fcf5 100644 --- a/pywikibot/exceptions.py +++ b/pywikibot/exceptions.py @@ -1,6 +1,43 @@ # -*- coding: utf-8 -*- """ Exception classes used throughout the framework. + +Error: Base class, all exceptions should the subclass of this class. + - NoUsername: Username is not in user-config.py + - UserBlockedY: our username or IP has been blocked + - AutoblockUser: requested action on a virtual autoblock user not valid + - UserActionRefuse + - NoSuchSite: Site does not exist + - BadTitle: Server responded with BadTitle + - InvalidTitle: Invalid page title + - PageNotFound: Page not found in list + - CaptchaError: Captcha is asked and config.solve_captcha == False + - Server504Error: Server timed out with HTTP 504 code + +PageRelatedError: any exception which is caused by an operation on a Page. + - NoPage: Page does not exist + - IsRedirectPage: Page is a redirect page + - IsNotRedirectPage: Page is not a redirect page + - CircularRedirect: Page is a circular redirect + - SectionError: The section specified by # does not exist + - LockedPage: Page is locked + - LockedNoPage: Title is locked against creation + - CascadeLockedPage: Page is locked due to cascading protection + +PageSaveRelatedError: page exceptions within the save operation on a Page. + (alias: PageNotSaved) + - SpamfilterError: MediaWiki spam filter detected a blacklisted URL + - OtherPageSaveError: misc. other save related exception. + - EditConflict: Edit conflict while uploading the page + - PageDeletedConflict: Page was deleted since being retrieved + - PageCreatedConflict: Page was created by another user + +ServerError: a problem with the server. + - FatalServerError: A fatal/non-recoverable server error + +WikiBaseError: any issue specific to Wikibase. + - CoordinateGlobeUnknownException: globe is not implemented yet. + """ # # (C) Pywikibot team, 2008 @@ -29,25 +66,72 @@ class PageRelatedError(Error): - """Abstract Exception, used when the Exception concerns a particular - Page, and when a generic message can be written once for all""" + """ + Abstract Exception, used when the exception concerns a particular Page. + """ + # Preformated UNICODE message where the page title will be inserted # Override this in subclasses. # u"Oh noes! Page %s is too funky, we should not delete it ;(" message = None - def __init__(self, page): + def __init__(self, page, message=None): """ - @param page + Constructor. + + @param page: Page that caused the exception @type page: Page object """ + if message: + self.message = message + if self.message is None: raise Error("PageRelatedError is abstract. Can't instantiate it!") - super(PageRelatedError, self).__init__(self.message % page) - self._page = page + + self.page = page + self.title = page.title(asLink=True) + self.site = page.site + + if '%(' in self.message and ')s' in self.message: + super(PageRelatedError, self).__init__(self.message % self.__dict__) + else: + super(PageRelatedError, self).__init__(self.message % page) def getPage(self): return self._page + + +class PageSaveRelatedError(PageRelatedError): + + """Saving the page has failed""" + message = u"Page %s was not saved." + + # This property maintains backwards compatibility with + # the old PageNotSaved which inherited from Error + # (not PageRelatedError) and exposed the normal 'args' + # which could be printed + @property + def args(self): + return unicode(self) + + +class OtherPageSaveError(PageSaveRelatedError): + + """Saving the page has failed due to uncatchable error.""" + message = "Edit to page %(title)s failed:\n%(reason)s" + + def __init__(self, page, reason): + """ Constructor. + + @param reason: Details of the problem + @type reason: Exception or basestring + """ + self.reason = reason + super(OtherPageSaveError, self).__init__(page) + + @property + def args(self): + return unicode(self.reason) class NoUsername(Error): @@ -95,10 +179,22 @@ """Invalid page title""" -class LockedPage(PageRelatedError): +class LockedPage(PageSaveRelatedError): """Page is locked""" message = u"Page %s is locked." + + +class LockedNoPage(LockedPage): + + """Title is locked against creation""" + message = u"Page %s does not exist and is locked preventing creation." + + +class CascadeLockedPage(LockedPage): + + """Page is locked due to cascading protection""" + message = u"Page %s is locked due to cascading protection." class SectionError(Error): @@ -106,26 +202,38 @@ """The section specified by # does not exist""" -class PageNotSaved(Error): - - """Saving the page has failed""" +PageNotSaved = PageSaveRelatedError -class EditConflict(PageNotSaved): +class EditConflict(PageSaveRelatedError): """There has been an edit conflict while uploading the page""" + message = u"Page %s could not be saved due to an edit conflict" -class SpamfilterError(PageNotSaved): +class PageDeletedConflict(EditConflict): + + """Page was deleted since being retrieved""" + message = u"Page %s has been deleted since last retrieved." + + +class PageCreatedConflict(EditConflict): + + """Page was created by another user""" + message = u"Page %s has been created since last retrieved." + + +class SpamfilterError(PageSaveRelatedError): """Saving the page has failed because the MediaWiki spam filter detected a blacklisted URL. """ - def __init__(self, arg): - super(SpamfilterError, self).__init__( - u'MediaWiki spam filter has been triggered') - self.url = arg - self.args = arg, + + message = "Edit to page %(title)s rejected by spam filter due to content:\n%(url)s" + + def __init__(self, page, url): + self.url = url + super(SpamfilterError, self).__init__(page) class ServerError(Error): diff --git a/pywikibot/page.py b/pywikibot/page.py index d30e649..e81aae5 100644 --- a/pywikibot/page.py +++ b/pywikibot/page.py @@ -960,9 +960,8 @@ else: watchval = "unwatch" if not force and not self.botMayEdit(): - raise pywikibot.PageNotSaved( - "Page %s not saved; editing restricted by {{bots}} template" - % self.title(asLink=True)) + raise pywikibot.OtherPageSaveError( + self, "Editing restricted by {{bots}} template") if botflag is None: botflag = ("bot" in self.site.userinfo["rights"]) if async: @@ -986,20 +985,17 @@ watch=watchval, bot=botflag, **kwargs) if not done: pywikibot.warning(u"Page %s not saved" % link) - raise pywikibot.PageNotSaved(link) + raise pywikibot.PageNotSaved(self) else: pywikibot.output(u"Page %s saved" % link) - except pywikibot.LockedPage as err: - # re-raise the LockedPage exception so that calling program - # can re-try if appropriate - if not callback and not async: - raise # TODO: other "expected" error types to catch? except pywikibot.Error as err: pywikibot.log(u"Error saving page %s (%s)\n" % (link, err), exc_info=True) if not callback and not async: - raise pywikibot.PageNotSaved("%s: %s" % (link, err)) + if isinstance(err, pywikibot.PageSaveRelatedError): + raise err + raise pywikibot.OtherPageSaveError(self, err) if callback: callback(self, err) diff --git a/pywikibot/site.py b/pywikibot/site.py index 07a280c..d47fda3 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -33,9 +33,14 @@ from pywikibot.throttle import Throttle from pywikibot.data import api from pywikibot.exceptions import ( - EditConflict, Error, + PageSaveRelatedError, + EditConflict, + PageCreatedConflict, + PageDeletedConflict, LockedPage, + CascadeLockedPage, + LockedNoPage, NoPage, NoSuchSite, NoUsername, @@ -3373,23 +3378,26 @@ rngen.request["grnredirect"] = "" return rngen - # catalog of editpage error codes, for use in generating messages + # Catalog of editpage error codes, for use in generating messages. + # The block at the bottom are page related errors. _ep_errors = { "noapiwrite": "API editing not enabled on %(site)s wiki", "writeapidenied": "User %(user)s is not authorized to edit on %(site)s wiki", - "protectedtitle": "Title %(title)s is protected against creation on %(site)s", "cantcreate": "User %(user)s not authorized to create new pages on %(site)s wiki", "cantcreate-anon": """Bot is not logged in, and anon users are not authorized to create new pages on %(site)s wiki""", - "articleexists": "Page %(title)s already exists on %(site)s wiki", "noimageredirect-anon": """Bot is not logged in, and anon users are not authorized to create image redirects on %(site)s wiki""", "noimageredirect": "User %(user)s not authorized to create image redirects on %(site)s wiki", - "spamdetected": "Edit to page %(title)s rejected by spam filter due to content:\n", "filtered": "%(info)s", "contenttoobig": "%(info)s", "noedit-anon": """Bot is not logged in, and anon users are not authorized to edit on %(site)s wiki""", "noedit": "User %(user)s not authorized to edit pages on %(site)s wiki", - "pagedeleted": "Page %(title)s has been deleted since last retrieved from %(site)s wiki", - "editconflict": "Page %(title)s not saved due to edit conflict.", + + "editconflict": EditConflict, + "articleexists": PageCreatedConflict, + "pagedeleted": PageDeletedConflict, + "protectedpage": LockedPage, + "protectedtitle": LockedNoPage, + "cascadeprotected": CascadeLockedPage, } @must_be(group='user') @@ -3437,8 +3445,7 @@ # before the page is saved. self.lock_page(page) if lastrev is not None and page.latestRevision() != lastrev: - raise EditConflict( - "editpage: Edit conflict detected; saving aborted.") + raise EditConflict(page) params = dict(action="edit", title=page.title(withSection=False), text=text, token=token, summary=summary) @@ -3477,23 +3484,17 @@ u"editpage: received '%s' even though bot is logged in" % err.code, _logger) - errdata = { - 'site': self, - 'title': page.title(withSection=False), - 'user': self.user(), - 'info': err.info - } - if err.code == "spamdetected": - raise SpamfilterError( - self._ep_errors[err.code] % errdata - + err.info[err.info.index("fragment: ") + 9:]) - - if err.code == "editconflict": - raise EditConflict(self._ep_errors[err.code] % errdata) - if err.code in ("protectedpage", "cascadeprotected"): - raise LockedPage(errdata['title']) if err.code in self._ep_errors: - raise Error(self._ep_errors[err.code] % errdata) + if issubclass(self._ep_errors[err.code], PageSaveRelatedError): + raise self._ep_errors[err.code](page) + else: + errdata = { + 'site': self, + 'title': page.title(withSection=False), + 'user': self.user(), + 'info': err.info + } + raise Error(self._ep_errors[err.code] % errdata) pywikibot.debug( u"editpage: Unexpected error code '%s' received." % err.code, @@ -3537,6 +3538,8 @@ u"page not saved" % captcha) return False + elif 'spamblacklist' in result['edit']: + raise SpamfilterError(page, result['edit']['spamblacklist']) else: self.unlock_page(page) pywikibot.error(u"editpage: unknown failure reason %s" @@ -3600,8 +3603,9 @@ raise Error("Cannot move page %s to its own title." % oldtitle) if not page.exists(): - raise NoPage("Cannot move page %s because it does not exist on %s." - % (oldtitle, self)) + raise NoPage(page, + "Cannot move page %(page)s because it " + "does not exist on %(site)s.") token = self.tokens['move'] self.lock_page(page) req = api.Request(site=self, action="move", to=newtitle, diff --git a/scripts/add_text.py b/scripts/add_text.py index b3cd488..1d104e1 100644 --- a/scripts/add_text.py +++ b/scripts/add_text.py @@ -260,13 +260,13 @@ u'Cannot change %s because of blacklist entry %s' % (page.title(), e.url)) return (False, False, always) - except pywikibot.PageNotSaved as error: - pywikibot.output(u'Error putting page: %s' % error.args) - return (False, False, always) except pywikibot.LockedPage: pywikibot.output(u'Skipping %s (locked page)' % page.title()) return (False, False, always) + except pywikibot.PageNotSaved as error: + pywikibot.output(u'Error putting page: %s' % error.args) + return (False, False, always) else: # Break only if the errors are one after the other... errorCount = 0 diff --git a/scripts/blockpageschecker.py b/scripts/blockpageschecker.py index 0417458..70c5fdf 100755 --- a/scripts/blockpageschecker.py +++ b/scripts/blockpageschecker.py @@ -474,14 +474,14 @@ u'blacklist entry %s' % (page.title(), e.url)) break - except pywikibot.PageNotSaved as error: - pywikibot.output(u'Error putting page: %s' - % (error.args,)) - break except pywikibot.LockedPage: pywikibot.output(u'The page is still protected. ' u'Skipping...') break + except pywikibot.PageNotSaved as error: + pywikibot.output(u'Error putting page: %s' + % (error.args,)) + break else: # Break only if the errors are one after the other errorCount = 0 diff --git a/scripts/reflinks.py b/scripts/reflinks.py index 2d35316..b22636a 100644 --- a/scripts/reflinks.py +++ b/scripts/reflinks.py @@ -744,11 +744,11 @@ pywikibot.output( u'Cannot change %s because of blacklist entry %s' % (page.title(), e.url)) - except pywikibot.PageNotSaved as error: - pywikibot.error(u'putting page: %s' % (error.args,)) except pywikibot.LockedPage: pywikibot.output(u'Skipping %s (locked page)' % page.title()) + except pywikibot.PageNotSaved as error: + pywikibot.error(u'putting page: %s' % (error.args,)) except pywikibot.ServerError as e: pywikibot.output(u'Server Error : %s' % e) diff --git a/scripts/replace.py b/scripts/replace.py index a8e6b46..9d92a1e 100755 --- a/scripts/replace.py +++ b/scripts/replace.py @@ -412,12 +412,12 @@ pywikibot.output( u'Cannot change %s because of blacklist entry %s' % (page.title(), e.url)) - except pywikibot.PageNotSaved as error: - pywikibot.output(u'Error putting page: %s' - % (error.args,)) except pywikibot.LockedPage: pywikibot.output(u'Skipping %s (locked page)' % (page.title(),)) + except pywikibot.PageNotSaved as error: + pywikibot.output(u'Error putting page: %s' + % (error.args,)) def prepareRegexForMySQL(pattern): diff --git a/tests/edit_failure_tests.py b/tests/edit_failure_tests.py new file mode 100644 index 0000000..50010b8 --- /dev/null +++ b/tests/edit_failure_tests.py @@ -0,0 +1,55 @@ +# -*- coding: utf-8 -*- +""" +Tests for edit failures. + +These tests should never write to the wiki, +unless something has broken badly. +""" +# +# (C) Pywikibot team, 2014 +# +# Distributed under the terms of the MIT license. +# +__version__ = '$Id$' + +import pywikibot +from pywikibot import ( + LockedPage, + SpamfilterError, + OtherPageSaveError, +) +from tests.utils import SiteTestCase, unittest + + +class TestSaveFailure(SiteTestCase): + """Test cases for edits which should fail to save.""" + + write = True + + def setUp(self): + super(TestSaveFailure, self).setUp() + self.site = pywikibot.Site('test', 'wikipedia') + + def test_protected(self): + """Test that protected titles raise the appropriate exception.""" + if self.site._username[1]: + raise unittest.SkipTest('Testing failure of edit protected with a sysop account') + page = pywikibot.Page(self.site, 'Wikipedia:Create a new page') + self.assertRaises(LockedPage, page.save) + + def test_spam(self): + """Test that spam in content raise the appropriate exception.""" + page = pywikibot.Page(self.site, 'Wikipedia:Sandbox') + page.text = 'http://badsite.com' + self.assertRaisesRegexp(SpamfilterError, 'badsite.com', page.save) + + def test_nobots(self): + """Test that {{nobots}} raise the appropriate exception.""" + page = pywikibot.Page(self.site, 'User:John Vandenberg/nobots') + self.assertRaisesRegexp(OtherPageSaveError, 'nobots', page.save) + +if __name__ == '__main__': + try: + unittest.main() + except SystemExit: + pass -- To view, visit https://gerrit.wikimedia.org/r/141398 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Id8b8c8055100329e7844b5db6500f71bbb1673f7 Gerrit-PatchSet: 11 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] [IMPROV] Missing Python 3 imports - change (pywikibot/core)
by jenkins-bot (Code Review) 06 Sep '14

06 Sep '14

jenkins-bot has submitted this change and it was merged. Change subject: [IMPROV] Missing Python 3 imports ...................................................................... [IMPROV] Missing Python 3 imports Change-Id: Iac3864b7fc15fe59645acde4e92ab52330e56c8f --- M pywikibot/compat/query.py M pywikibot/diff.py M pywikibot/page.py M pywikibot/site.py M pywikibot/userinterfaces/gui.py M pywikibot/version.py M scripts/category_redirect.py M scripts/checkimages.py M scripts/data_ingestion.py M scripts/flickrripper.py M scripts/imagerecat.py M scripts/maintenance/wikimedia_sites.py M scripts/makecat.py M scripts/reflinks.py M scripts/script_wui.py M scripts/upload.py M scripts/weblinkchecker.py M scripts/welcome.py 18 files changed, 152 insertions(+), 70 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/compat/query.py b/pywikibot/compat/query.py index a23025a..42ff2c7 100644 --- a/pywikibot/compat/query.py +++ b/pywikibot/compat/query.py @@ -2,6 +2,12 @@ from pywikibot.data import api from pywikibot.tools import deprecated, deprecate_arg +import sys +if sys.version_info[0] > 2: + import io as StringIO +else: + import StringIO + @deprecated("pywikibot.data.api.Request") @deprecate_arg("useAPI", None) @@ -17,7 +23,6 @@ if back_response: pywikibot.warning(u"back_response is no longer supported; an empty " u"response object will be returned") - import StringIO res_dummy = StringIO.StringIO() res_dummy.__dict__.update({u'code': 0, u'msg': u''}) return res_dummy, result diff --git a/pywikibot/diff.py b/pywikibot/diff.py index 490b0df..346fcba 100644 --- a/pywikibot/diff.py +++ b/pywikibot/diff.py @@ -11,7 +11,11 @@ import difflib -import itertools +import sys +if sys.version_info[0] > 2: + from itertools import zip_longest +else: + from itertools import izip_longest as zip_longest import pywikibot @@ -129,7 +133,7 @@ colored_line = u'' state = 'Close' - for char, char_ref in itertools.izip_longest(line, line_ref.strip(), fillvalue=' '): + for char, char_ref in zip_longest(line, line_ref.strip(), fillvalue=' '): char_tagged = char if state == 'Close': if char_ref != ' ': diff --git a/pywikibot/page.py b/pywikibot/page.py index d30e649..ca1d169 100644 --- a/pywikibot/page.py +++ b/pywikibot/page.py @@ -29,18 +29,18 @@ if sys.version_info[0] == 2: import htmlentitydefs from urllib import quote as quote_from_bytes, unquote as unquote_to_bytes + from urllib import urlopen else: unicode = basestring = str from html import entities as htmlentitydefs from urllib.parse import quote_from_bytes, unquote_to_bytes + from urllib.request import urlopen import logging import re import sys import unicodedata import collections - -import urllib logger = logging.getLogger("pywiki.wiki.page") @@ -1878,7 +1878,7 @@ """Return image file's MD5 checksum.""" # FIXME: MD5 might be performed on incomplete file due to server disconnection # (see bug #1795683). - f = urllib.urlopen(self.fileUrl()) + f = urlopen(self.fileUrl()) # TODO: check whether this needs a User-Agent header added h = hashlib.md5() h.update(f.read()) diff --git a/pywikibot/site.py b/pywikibot/site.py index 07a280c..14954d5 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -23,7 +23,6 @@ from collections import Iterable, Container import threading import time -import urllib import json import copy @@ -45,8 +44,11 @@ from pywikibot.echo import Notification if sys.version_info[0] > 2: + from urllib.parse import urlencode basestring = (str,) unicode = str +else: + from urllib import urlencode _logger = "wiki.site" @@ -811,7 +813,7 @@ @deprecated("urllib.urlencode()") def urlEncode(self, query): """DEPRECATED.""" - return urllib.urlencode(query) + return urlencode(query) @deprecated("pywikibot.comms.http.request") def getUrl(self, path, retry=True, sysop=False, data=None, @@ -825,7 +827,7 @@ from pywikibot.comms import http if data: if not isinstance(data, basestring): - data = urllib.urlencode(data) + data = urlencode(data) return http.request(self, path, method="PUT", body=data) else: return http.request(self, path) diff --git a/pywikibot/userinterfaces/gui.py b/pywikibot/userinterfaces/gui.py index abdbe76..db7f0c3 100644 --- a/pywikibot/userinterfaces/gui.py +++ b/pywikibot/userinterfaces/gui.py @@ -16,9 +16,15 @@ __version__ = '$Id$' # -import Tkinter -from ScrolledText import ScrolledText -import tkSimpleDialog +import sys +if sys.version_info[0] > 2: + import tkinter as Tkinter + from tkinter.scrolledtext import ScrolledText + from tkinter import simpledialog as tkSimpleDialog +else: + import Tkinter + from ScrolledText import ScrolledText + import tkSimpleDialog from idlelib import SearchDialog, ReplaceDialog, configDialog from idlelib.configHandler import idleConf diff --git a/pywikibot/version.py b/pywikibot/version.py index 9243910..8fc5685 100644 --- a/pywikibot/version.py +++ b/pywikibot/version.py @@ -14,6 +14,7 @@ import time import datetime import subprocess +import sys import pywikibot.config2 as config @@ -145,7 +146,10 @@ @return: the git hash @rtype: str """ - from StringIO import StringIO + if sys.version_info[0] > 2: + from io import StringIO + else: + from StringIO import StringIO import xml.dom.minidom from pywikibot.comms import http diff --git a/scripts/category_redirect.py b/scripts/category_redirect.py index 5eb105a..b2123b2 100755 --- a/scripts/category_redirect.py +++ b/scripts/category_redirect.py @@ -21,8 +21,7 @@ # __version__ = '$Id$' # - -import cPickle +import sys import re import time from datetime import datetime, timedelta @@ -30,6 +29,11 @@ from pywikibot import pagegenerators from pywikibot import i18n +if sys.version_info[0] > 2: + import pickle as cPickle +else: + import cPickle + class CategoryRedirectBot(object): diff --git a/scripts/checkimages.py b/scripts/checkimages.py index 1343c6e..8004226 100644 --- a/scripts/checkimages.py +++ b/scripts/checkimages.py @@ -95,7 +95,6 @@ import time import datetime import locale -import urllib import pywikibot from pywikibot import pagegenerators as pg from pywikibot import config, i18n @@ -917,14 +916,8 @@ return imageName def convert_to_url(self, page): - # Function stolen from wikipedia.py - """The name of the page this Page refers to, in a form suitable for the - URL of the page. - - """ - title = page.replace(u" ", u"_") - encodedTitle = title.encode(self.site.encoding()) - return urllib.quote(encodedTitle) + """Return the page title suitable as for an URL.""" + return page.title(asUrl=True) def countEdits(self, pagename, userlist): """Function to count the edit of a user or a list of users in a page.""" diff --git a/scripts/data_ingestion.py b/scripts/data_ingestion.py index a791b0d..83ec1c0 100755 --- a/scripts/data_ingestion.py +++ b/scripts/data_ingestion.py @@ -10,14 +10,21 @@ # import posixpath -import urlparse -import urllib import hashlib import base64 -import StringIO +import sys import pywikibot import upload + +if sys.version_info[0] > 2: + from urllib.parse import urlparse + from urllib.request import urlopen + import io as StringIO +else: + from urlparse import urlparse + from urllib import urlopen + import StringIO class Photo(object): @@ -37,7 +44,7 @@ self.metadata = metadata self.metadata["_url"] = URL self.metadata["_filename"] = filename = posixpath.split( - urlparse.urlparse(URL)[2])[1] + urlparse(URL)[2])[1] self.metadata["_ext"] = ext = filename.split(".")[-1] if ext == filename: self.metadata["_ext"] = ext = None @@ -50,7 +57,7 @@ TODO: Add exception handling """ if not self.contents: - imageFile = urllib.urlopen(self.URL).read() + imageFile = urlopen(self.URL).read() self.contents = StringIO.StringIO(imageFile) return self.contents @@ -183,7 +190,7 @@ TODO: Add exception handling """ - imageFile = urllib.urlopen(photoUrl).read() + imageFile = urlopen(photoUrl).read() return StringIO.StringIO(imageFile) def findDuplicateImages(self, photo=None, site=pywikibot.Site(u'commons', u'commons')): diff --git a/scripts/flickrripper.py b/scripts/flickrripper.py index 37a8808..d5550d2 100644 --- a/scripts/flickrripper.py +++ b/scripts/flickrripper.py @@ -32,12 +32,11 @@ __version__ = '$Id$' # -import urllib import re -import StringIO import hashlib import base64 import time +import sys import pywikibot from pywikibot import config, textlib import upload @@ -45,16 +44,26 @@ try: import flickrapi # see: http://stuvel.eu/projects/flickrapi except ImportError: - import sys pywikibot.error('This script requires the python flickrapi module') pywikibot.error('See: http://stuvel.eu/projects/flickrapi') pywikibot.exception() sys.exit() -from Tkinter import ( - Tk, Label, Entry, Scrollbar, Text, Button, - END, VERTICAL, NORMAL, WORD -) +if sys.version_info[0] > 2: + from urllib.parse import urlencode + from urllib.request import urlopen + import io as StringIO + from tkinter import ( + Tk, Label, Entry, Scrollbar, Text, Button, + END, VERTICAL, NORMAL, WORD + ) +else: + from urllib import urlencode, urlopen + import StringIO + from Tkinter import ( + Tk, Label, Entry, Scrollbar, Text, Button, + END, VERTICAL, NORMAL, WORD + ) from PIL import Image, ImageTk # see: http://www.pythonware.com/products/pil/ flickr_allowed_license = { @@ -118,7 +127,7 @@ TODO: Add exception handling """ - imageFile = urllib.urlopen(photoUrl).read() + imageFile = urlopen(photoUrl).read() return StringIO.StringIO(imageFile) @@ -150,9 +159,9 @@ TODO: Add exception handling, try a couple of times """ - parameters = urllib.urlencode({'id': photo_id, 'raw': 'on'}) + parameters = urlencode({'id': photo_id, 'raw': 'on'}) - rawDescription = urllib.urlopen( + rawDescription = urlopen( "http://wikipedia.ramselehof.de/flinfo.php?%s" % parameters).read() return rawDescription.decode('utf-8') diff --git a/scripts/imagerecat.py b/scripts/imagerecat.py index fbeee07..295bb7f 100644 --- a/scripts/imagerecat.py +++ b/scripts/imagerecat.py @@ -34,13 +34,20 @@ # import re -import urllib import time import socket import xml.etree.ElementTree +import sys import pywikibot from pywikibot import pagegenerators, textlib + +if sys.version_info[0] > 2: + from urllib.parse import urlencode + from urllib.request import urlopen +else: + from urllib import urlencode, urlopen + category_blacklist = [] countries = [] @@ -119,14 +126,14 @@ lang = site.language() family = site.family.name if lang == u'commons' and family == u'commons': - parameters = urllib.urlencode( + parameters = urlencode( {'i': imagepage.title(withNamespace=False).encode('utf-8'), 'r': 'on', 'go-clean': 'Find+Categories', 'p': search_wikis, 'cl': hint_wiki}) elif family == u'wikipedia': - parameters = urllib.urlencode( + parameters = urlencode( {'i': imagepage.title(withNamespace=False).encode('utf-8'), 'r': 'on', 'go-move': 'Find+Categories', @@ -147,7 +154,7 @@ try: if tries < maxtries: tries += 1 - commonsHelperPage = urllib.urlopen( + commonsHelperPage = urlopen( "https://toolserver.org/~daniel/WikiSense/CommonSense.php?%s" % parameters) matches = commonsenseRe.search( commonsHelperPage.read().decode('utf-8')) @@ -210,10 +217,10 @@ """ result = [] gotInfo = False - parameters = urllib.urlencode({'lat': latitude, 'lon': longitude, 'accept-language': 'en'}) + parameters = urlencode({'lat': latitude, 'lon': longitude, 'accept-language': 'en'}) while not gotInfo: try: - page = urllib.urlopen("https://nominatim.openstreetmap.org/reverse?format=xml&%s" % parameters) + page = urlopen("https://nominatim.openstreetmap.org/reverse?format=xml&%s" % parameters) et = xml.etree.ElementTree.parse(page) gotInfo = True except IOError: @@ -366,11 +373,11 @@ for cat in categories: cat = cat.replace('_', ' ') toFilter = toFilter + "[[Category:" + cat + "]]\n" - parameters = urllib.urlencode({'source': toFilter.encode('utf-8'), + parameters = urlencode({'source': toFilter.encode('utf-8'), 'bot': '1'}) filterCategoriesRe = re.compile('\[\[Category:([^\]]*)\]\]') try: - filterCategoriesPage = urllib.urlopen( + filterCategoriesPage = urlopen( "https://toolserver.org/~multichill/filtercats.php?%s" % parameters) result = filterCategoriesRe.findall( filterCategoriesPage.read().decode('utf-8')) diff --git a/scripts/maintenance/wikimedia_sites.py b/scripts/maintenance/wikimedia_sites.py index 76a9361..fa94029 100644 --- a/scripts/maintenance/wikimedia_sites.py +++ b/scripts/maintenance/wikimedia_sites.py @@ -14,11 +14,16 @@ import re import codecs -import urllib from xml.etree import cElementTree +import sys import pywikibot from pywikibot.site import Family + +if sys.version_info[0] > 2: + from urllib.request import urlopen +else: + from urllib import urlopen URL = 'https://wikistats.wmflabs.org/api.php?action=dump&table=%s&format=xml' @@ -44,7 +49,7 @@ original = Family(family).languages_by_size obsolete = Family(family).obsolete - feed = urllib.urlopen(URL % familiesDict[family]) + feed = urlopen(URL % familiesDict[family]) tree = cElementTree.parse(feed) new = [] diff --git a/scripts/makecat.py b/scripts/makecat.py index f1c8a7e..e523cff 100644 --- a/scripts/makecat.py +++ b/scripts/makecat.py @@ -122,6 +122,8 @@ pywikibot.output(u'') pywikibot.output(u"==%s==" % pl.title()) while True: + # TODO: Use pywikibot.inputChoice? + # (needs the support for 'other options') answer = pywikibot.input("[y]es/[n]o/[i]gnore/[o]ther options?") if answer == 'y': include(pl) diff --git a/scripts/reflinks.py b/scripts/reflinks.py index 2d35316..11031fd 100644 --- a/scripts/reflinks.py +++ b/scripts/reflinks.py @@ -43,19 +43,29 @@ # import re -import urllib2 -import httplib import socket import codecs import subprocess import tempfile import os import gzip -import StringIO +import sys import pywikibot from pywikibot import i18n, pagegenerators, textlib, xmlreader, Bot import noreferences + +# TODO: Convert to httlib2 +if sys.version_info[0] > 2: + from urllib.parse import quote + from urllib.request import urlopen + from urllib.error import HTTPError, URLError + import http.client as httplib + import io as StringIO +else: + from urllib2 import quote, urlopen, HTTPError, URLError + import httplib + import StringIO docuReplacements = { '&params;': pagegenerators.parameterHelp @@ -528,10 +538,10 @@ try: socket.setdefaulttimeout(20) try: - f = urllib2.urlopen(ref.url.decode("utf8")) + f = urlopen(ref.url.decode("utf8")) except UnicodeError: - ref.url = urllib2.quote(ref.url.encode("utf8"), "://") - f = urllib2.urlopen(ref.url) + ref.url = quote(ref.url.encode("utf8"), "://") + f = urlopen(ref.url) # Try to get Content-Type from server headers = f.info() contentType = headers.getheader('Content-Type') @@ -596,7 +606,7 @@ u'\03{lightred}Bad link\03{default} : %s in %s' % (ref.url, page.title(asLink=True))) continue - except urllib2.HTTPError as e: + except HTTPError as e: pywikibot.output(u'HTTP error (%s) for %s on %s' % (e.code, ref.url, page.title(asLink=True)), @@ -608,7 +618,7 @@ repl = ref.refDead() new_text = new_text.replace(match.group(), repl) continue - except (urllib2.URLError, + except (URLError, socket.error, IOError, httplib.error) as e: diff --git a/scripts/script_wui.py b/scripts/script_wui.py index ab2a595..1e2ef0c 100755 --- a/scripts/script_wui.py +++ b/scripts/script_wui.py @@ -62,7 +62,6 @@ import datetime -import thread import threading import sys import os @@ -84,6 +83,11 @@ import pywikibot # pywikibot.botirc depends on https://pypi.python.org/pypi/irc import pywikibot.botirc + +if sys.version_info[0] > 2: + import _thread as thread +else: + import thread bot_config = { @@ -203,7 +207,10 @@ def main_script(page, rev=None, params=None): # http://opensourcehacker.com/2011/02/23/temporarily-capturing-python-logging… # https://docs.python.org/release/2.6/library/logging.html - from StringIO import StringIO + if sys.version_info[0] > 2: + from io import StringIO + else: + from StringIO import StringIO import logging # safety; default mode is safe (no writing) diff --git a/scripts/upload.py b/scripts/upload.py index 9261661..17ae914 100755 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -40,16 +40,22 @@ import os import time -import urllib -import urlparse import tempfile import re import math +import sys import pywikibot import pywikibot.data.api from pywikibot import config from pywikibot.bot import QuitKeyboardInterrupt + +if sys.version_info[0] > 2: + from urllib.parse import urlparse + from urllib.request import URLopener +else: + from urlparse import urlparse + from urllib import URLopener class UploadRobot: @@ -91,7 +97,7 @@ rlen = 0 _contents = None dt = 15 - uo = urllib.URLopener() + uo = URLopener() retrieved = False while not retrieved: @@ -149,7 +155,7 @@ # Filename may be either a local file path or a URL if "://" in filename: # extract the path portion of the URL - filename = urlparse.urlparse(filename).path + filename = urlparse(filename).path filename = os.path.basename(filename) if self.useFilename: diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py index 1ead1d9..262d695 100644 --- a/scripts/weblinkchecker.py +++ b/scripts/weblinkchecker.py @@ -99,16 +99,24 @@ import re import codecs import pickle -import httplib import socket -import urlparse -import urllib import threading import time +import sys import pywikibot from pywikibot import i18n, config, pagegenerators, textlib, xmlreader, weblib +# TODO: Convert to httlib2 +if sys.version_info[0] > 2: + import urllib.parse as urlparse + import urllib.request as urllib + import http.client as httplib +else: + import urlparse + import urllib + import httplib + docuReplacements = { '&params;': pagegenerators.parameterHelp } diff --git a/scripts/welcome.py b/scripts/welcome.py index 0456bf6..1de697e 100644 --- a/scripts/welcome.py +++ b/scripts/welcome.py @@ -186,7 +186,7 @@ import locale import codecs from random import choice -from string import capitalize +import sys import pywikibot from pywikibot import i18n from pywikibot import config @@ -651,7 +651,7 @@ 'Log page is not exist, getting information for page creation') text = i18n.translate(self.site, logpage_header) text += u'\n!%s' % self.site.namespace(2) - text += u'\n!%s' % capitalize( + text += u'\n!%s' % str.capitalize( self.site.mediawiki_message('contribslink')) for result in queue: @@ -992,7 +992,10 @@ # If there is the savedata, the script must save the number_user. if globalvar.randomSign and globalvar.saveSignIndex and \ bot.welcomed_users: - import cPickle + if sys.version_info[0] > 2: + import pickle as cPickle + else: + import cPickle with open(filename, 'w') as f: cPickle.dump(bot.welcomed_users, f) -- To view, visit https://gerrit.wikimedia.org/r/158841 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Iac3864b7fc15fe59645acde4e92ab52330e56c8f Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] [FIX] Category: Only dump cache if loaded - change (pywikibot/core)
by jenkins-bot (Code Review) 06 Sep '14

06 Sep '14

jenkins-bot has submitted this change and it was merged. Change subject: [FIX] Category: Only dump cache if loaded ...................................................................... [FIX] Category: Only dump cache if loaded This also removes the 'loaded' variable as it's basically defined if both the attributes are present. This also set the content to 'loaded' after loading the file. Change-Id: I788166b45f5644a23d6d3dad858f8d32b6e16e38 --- M scripts/category.py 1 file changed, 7 insertions(+), 4 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/scripts/category.py b/scripts/category.py index 68dfc9d..246d637 100755 --- a/scripts/category.py +++ b/scripts/category.py @@ -141,12 +141,16 @@ if not os.path.isabs(filename): filename = config.datafilepath(filename) self.filename = filename - self.loaded = False if rebuild: self.rebuild() + @property + def is_loaded(self): + """Return whether the contents have been loaded.""" + return hasattr(self, 'catContentDB') and hasattr(self, 'superclassDB') + def _load(self): - if not self.loaded: + if not self.is_loaded(): try: f = bz2.BZ2File(self.filename, 'r') pywikibot.output(u'Reading dump from %s' @@ -166,7 +170,6 @@ def rebuild(self): self.catContentDB = {} self.superclassDB = {} - self.loaded = True def getSubcats(self, supercat): """Return the list of subcategories for a given supercategory. @@ -226,7 +229,7 @@ filename = self.filename elif not os.path.isabs(filename): filename = config.datafilepath(filename) - if self.catContentDB or self.superclassDB: + if self.is_loaded and (self.catContentDB or self.superclassDB): pywikibot.output(u'Dumping to %s, please wait...' % config.shortpath(filename)) f = bz2.BZ2File(filename, 'w') -- To view, visit https://gerrit.wikimedia.org/r/158844 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I788166b45f5644a23d6d3dad858f8d32b6e16e38 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] movepages.py unable to login without password.lwp - change (pywikibot/core)
by jenkins-bot (Code Review) 05 Sep '14

05 Sep '14

jenkins-bot has submitted this change and it was merged. Change subject: movepages.py unable to login without password.lwp ...................................................................... movepages.py unable to login without password.lwp Site.movepage must have user rights. Added missing @must_be(group='user'), which forces login(). Bug 70447 Change-Id: I09061c0a8e4fb80362eccddf00cb713fe83aabe4 --- M pywikibot/site.py 1 file changed, 1 insertion(+), 0 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/site.py b/pywikibot/site.py index 07a280c..ad3ae8d 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -3576,6 +3576,7 @@ "[[%(newtitle)s]] file extension does not match content of [[%(oldtitle)s]]" } + @must_be(group='user') def movepage(self, page, newtitle, summary, movetalk=True, noredirect=False): """Move a Page to a new title. -- To view, visit https://gerrit.wikimedia.org/r/158805 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I09061c0a8e4fb80362eccddf00cb713fe83aabe4 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] big improvements to the tokens storage - change (pywikibot/core)
by jenkins-bot (Code Review) 05 Sep '14

05 Sep '14

jenkins-bot has submitted this change and it was merged. Change subject: big improvements to the tokens storage ...................................................................... big improvements to the tokens storage - new APISite._tokens property (for internal use) which holds tokens cached *by user* - new APISite.tokens property (for external use) of TokenWallet type - new APISite.preload_tokens() method for caching tokens - use action=tokens instead of action=query&prop=info&intoken=etc. when supported - the APISite.token() method has been deprecated and replaced with the 'tokens' property also added a very simple test Change-Id: I805002455798843fbe77556d3a6134bc9434417e --- M pywikibot/site.py M tests/site_tests.py 2 files changed, 66 insertions(+), 39 deletions(-) Approvals: Legoktm: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/site.py b/pywikibot/site.py index 46995af..07a280c 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -1155,6 +1155,21 @@ return None +class TokenWallet(object): + + """Container for tokens.""" + + def __init__(self, site): + self.site = site + self.site._tokens = {} + + def __getitem__(self, key): + storage = self.site._tokens.setdefault(self.site.user(), {}) + if key not in storage: + self.site.preload_tokens([key]) + return storage[key] + + class APISite(BaseSite): """API interface to MediaWiki site. @@ -1190,7 +1205,7 @@ self._msgcache = {} self._loginstatus = LoginStatus.NOT_ATTEMPTED self._siteinfo = Siteinfo(self) - return + self.tokens = TokenWallet(self) @staticmethod def fromDBName(dbname): @@ -2172,6 +2187,35 @@ api.update_page(page, pagedata) yield page + def preload_tokens(self, types): + """Preload one or multiple tokens. + + For all MediaWiki versions prior to 1.20, only one token can be + retrieved at once. + + @param types: the types of token (e.g., "edit", "move", "delete"); + see API documentation for full list of types + @type types: iterable + """ + storage = self._tokens.setdefault(self.user(), {}) + if LV(self.version()) < LV('1.20'): + for tokentype in types: + query = api.PropertyGenerator('info', + titles='Dummy page', + intoken=tokentype, + site=self) + for item in query: + pywikibot.debug(unicode(item), _logger) + if (tokentype + 'token') in item: + storage[tokentype] = item[tokentype + 'token'] + else: + data = api.Request(site=self, action='tokens', + type='|'.join(types)).submit() + if 'tokens' in data and data['tokens']: + storage.update(dict((key[:-5], val) + for key, val in data['tokens'].items())) + + @deprecated("the 'tokens' property") def token(self, page, tokentype): """Return token retrieved from wiki to allow changing page content. @@ -2180,19 +2224,7 @@ see API documentation for full list of types """ - query = api.PropertyGenerator("info", - titles=page.title(withSection=False), - intoken=tokentype, - site=self) - for item in query: - if not self.sametitle(item['title'], page.title(withSection=False)): - raise Error( - u"token: Query on page %s returned data on page [[%s]]" - % (page.title(withSection=False, asLink=True), - item['title'])) - api.update_page(page, item) - pywikibot.debug(unicode(item), _logger) - return item[tokentype + "token"] + return self.tokens[tokentype] # following group of methods map more-or-less directly to API queries @@ -3396,7 +3428,7 @@ lastrev = None if not recreate: raise - token = self.token(page, "edit") + token = self.tokens['edit'] # getting token also updates the 'lastrevid' value, which allows us to # detect if page has been changed since last time text was retrieved. @@ -3570,7 +3602,7 @@ if not page.exists(): raise NoPage("Cannot move page %s because it does not exist on %s." % (oldtitle, self)) - token = self.token(page, "move") + token = self.tokens['move'] self.lock_page(page) req = api.Request(site=self, action="move", to=newtitle, token=token, reason=summary) @@ -3688,7 +3720,7 @@ @param summary: Edit summary (required!). """ - token = self.token(page, "delete") + token = self.tokens['delete'] self.lock_page(page) req = api.Request(site=self, action="delete", token=token, title=page.title(withSection=False), @@ -3758,7 +3790,7 @@ @type expiry: pywikibot.Timestamp, string in GNU timestamp format (including ISO 8601). """ - token = self.token(page, 'protect') + token = self.tokens['protect'] self.lock_page(page) protectList = [ptype + '=' + level for ptype, level in protections.items() @@ -3796,7 +3828,7 @@ def blockuser(self, user, expiry, reason, anononly=True, nocreate=True, autoblock=True, noemail=False, reblock=False): - token = self.token(user, 'block') + token = self.tokens['block'] if isinstance(expiry, pywikibot.Timestamp): expiry = expiry.toISOformat() req = api.Request(site=self, action='block', user=user.username, @@ -3818,7 +3850,7 @@ @must_be(group='sysop') def unblockuser(self, user, reason): - token = self.token(user, 'block') + token = self.tokens['block'] req = api.Request(site=self, action='unblock', user=user.username, reason=reason, token=token) @@ -3833,7 +3865,7 @@ @return: True if API returned expected response; False otherwise """ - token = self.token(page, "watch") + token = self.tokens['watch'] req = api.Request(action="watch", token=token, title=page.title(withSection=False)) if unwatch: @@ -3954,7 +3986,7 @@ text = filepage.text if not text: text = comment - token = self.token(filepage, "edit") + token = self.tokens['edit'] result = None if source_filename: # upload local file @@ -4436,8 +4468,7 @@ params['bot'] = 1 if 'baserevid' in kwargs and kwargs['baserevid']: params['baserevid'] = kwargs['baserevid'] - params['token'] = self.token(pywikibot.Page(self, u'Main Page'), - 'edit') # Use a dummy page + params['token'] = self.tokens['edit'] for arg in kwargs: if arg in ['clear', 'data', 'exclude', 'summary']: params[arg] = kwargs[arg] @@ -4461,8 +4492,7 @@ params['value'] = json.dumps(claim._formatValue()) if 'summary' in kwargs: params['summary'] = kwargs['summary'] - params['token'] = self.token(pywikibot.Page(self, u'Main Page'), - 'edit') + params['token'] = self.tokens['edit'] req = api.Request(site=self, **params) data = req.submit() claim.snak = data['claim']['id'] @@ -4496,8 +4526,7 @@ params['bot'] = 1 if 'summary' in kwargs: params['summary'] = kwargs['summary'] - params['token'] = self.token(pywikibot.Page(self, u'Main Page'), - 'edit') + params['token'] = self.tokens['edit'] if snaktype == 'value': params['value'] = json.dumps(claim._formatValue()) @@ -4527,8 +4556,7 @@ params['baserevid'] = claim.on_item.lastrevid if bot: params['bot'] = 1 - params['token'] = self.token(pywikibot.Page(self, u'Main Page'), - 'edit') + params['token'] = self.tokens['edit'] # build up the snak if isinstance(source, list): sources = source @@ -4584,8 +4612,7 @@ hasattr(qualifier, 'hash') and qualifier.hash is not None): params['snakhash'] = qualifier.hash - params['token'] = self.token(pywikibot.Page(self, u'Main Page'), - 'edit') + params['token'] = self.tokens['edit'] # build up the snak if qualifier.getSnakType() == 'value': params['value'] = json.dumps(qualifier._formatValue()) @@ -4606,8 +4633,7 @@ if bot: params['bot'] = 1 params['claim'] = '|'.join(claim.snak for claim in claims) - params['token'] = self.token(pywikibot.Page(self, u'Main Page'), - 'edit') # Use a dummy page + params['token'] = self.tokens['edit'] for kwarg in kwargs: if kwarg in ['baserevid', 'summary']: params[kwarg] = kwargs[kwarg] @@ -4630,8 +4656,7 @@ params['bot'] = 1 params['statement'] = claim.snak params['references'] = '|'.join(source.hash for source in sources) - params['token'] = self.token(pywikibot.Page(self, u'Main Page'), - 'edit') # Use a dummy page + params['token'] = self.tokens['edit'] for kwarg in kwargs: if kwarg in ['baserevid', 'summary']: params[kwarg] = kwargs[kwarg] @@ -4656,7 +4681,7 @@ 'totitle': page1.title(), 'fromsite': page2.site.dbName(), 'fromtitle': page2.title(), - 'token': self.token(page1, 'edit') + 'token': self.tokens['edit'] } if bot: params['bot'] = 1 @@ -4678,7 +4703,7 @@ 'action': 'wbmergeitems', 'fromid': fromItem.getID(), 'toid': toItem.getID(), - 'token': self.token(toItem, 'edit') + 'token': self.tokens['edit'] } for kwarg in kwargs: if kwarg in ['ignoreconflicts', 'summary']: diff --git a/tests/site_tests.py b/tests/site_tests.py index 125b6aa..da4b609 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -228,7 +228,9 @@ def testTokens(self): """Test ability to get page tokens.""" for ttype in ("edit", "move"): # token types for non-sysops - self.assertIsInstance(mysite.token(mainpage, ttype), basestring) + token = mysite.token(mainpage, ttype) + self.assertIsInstance(token, basestring) + self.assertEqual(token, mysite.tokens[ttype]) self.assertRaises(KeyError, mysite.token, mainpage, "invalidtype") def testPreload(self): -- To view, visit https://gerrit.wikimedia.org/r/139372 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I805002455798843fbe77556d3a6134bc9434417e Gerrit-PatchSet: 13 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Ricordisamoa <ricordisamoa(a)openmailbox.org> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Legoktm <legoktm.wikipedia(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Ricordisamoa <ricordisamoa(a)openmailbox.org> Gerrit-Reviewer: Russell Blau <russblau(a)imapmail.org> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot <>

1 0

← Newer
1
...
11
12
13
14
15
16
17
18
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

Pywikibot-commits September 2014