Pywikibot-commits October 2014

pywikibot-commits@lists.wikimedia.org

1 participants
155 discussions

[Gerrit] Construct MIME form post in static method - change (pywikibot/core)
by jenkins-bot (Code Review) 13 Oct '14

13 Oct '14

jenkins-bot has submitted this change and it was merged. Change subject: Construct MIME form post in static method ...................................................................... Construct MIME form post in static method Moves MIME form post construction from Request.submit to static method Request._build_mime_request Request param 'file' has been a special case invoking MIME handling. Site.upload has been updated so param 'file' is a mime_param, and api.Request has no special case for it. Change-Id: I09abe7e8350c034f1d5a2c81d99063ab3c3f7924 --- M pywikibot/data/api.py M pywikibot/site.py 2 files changed, 57 insertions(+), 40 deletions(-) Approvals: John Vandenberg: Looks good to me, approved XZise: Looks good to me, but someone else must approve jenkins-bot: Verified diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index 17035be..dd321d4 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -14,7 +14,6 @@ import datetime import hashlib import json -import mimetypes import os try: import cPickle as pickle @@ -425,12 +424,12 @@ return message == ERR_MSG @staticmethod - def _generate_MIME_part(key, content, keytype, headers): + def _generate_MIME_part(key, content, keytype=None, headers=None): if not keytype: try: content.encode("ascii") keytype = ("text", "plain") - except UnicodeError: + except (UnicodeError, AttributeError): keytype = ("application", "octet-stream") submsg = MIMENonMultipart(*keytype) content_headers = {'name': key} @@ -440,6 +439,35 @@ **content_headers) submsg.set_payload(content) return submsg + + @staticmethod + def _build_mime_request(params, mime_params): + """Construct a MIME multipart form post. + + @param params: HTTP request params + @type params: dict + @param mime_params: HTTP request parts which must be sent in the body + @type mime_params: dict of (content, keytype, headers) + @return: HTTP request headers and body + @rtype: (headers, body) + """ + # construct a MIME message containing all API key/values + container = MIMEMultipart(_subtype='form-data') + for key, value in params.items(): + submsg = Request._generate_MIME_part(key, value) + container.attach(submsg) + for key, value in mime_params.items(): + submsg = Request._generate_MIME_part(key, *value) + container.attach(submsg) + + # strip the headers to get the HTTP message body + body = container.as_string() + marker = "\n\n" # separates headers from body + eoh = body.find(marker) + body = body[eoh + len(marker):] + # retrieve the headers from the MIME object + headers = dict(list(container.items())) + return headers, body def _handle_warnings(self, result): if 'warnings' in result: @@ -482,32 +510,8 @@ uri = self.site.scriptpath() + "/api.php" try: if self.mime: - # construct a MIME message containing all API key/values - container = MIMEMultipart(_subtype='form-data') - for key, value in self._encoded_items().items(): - # key "file" requires special treatment in a multipart - # message - if key == "file": - local_filename = value - filetype = mimetypes.guess_type(local_filename)[0] \ - or 'application/octet-stream' - file_content = file(local_filename, "rb").read() - submsg = Request._generate_MIME_part( - key, file_content, filetype.split('/'), - {'filename': self._params['filename'][0]}) - else: - submsg = Request._generate_MIME_part( - key, value, None, None) - container.attach(submsg) - for key, value in self.mime_params.items(): - container.attach(Request._generate_MIME_part(key, *value)) - # strip the headers to get the HTTP message body - body = container.as_string() - marker = "\n\n" # separates headers from body - eoh = body.find(marker) - body = body[eoh + len(marker):] - # retrieve the headers from the MIME object - headers = dict(list(container.items())) + (headers, body) = Request._build_mime_request( + self._encoded_items(), self.mime_params) else: headers = {'Content-Type': 'application/x-www-form-urlencoded'} body = paramstring diff --git a/pywikibot/site.py b/pywikibot/site.py index 600d941..43b781c 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -23,6 +23,7 @@ import time import json import copy +import mimetypes import pywikibot import pywikibot.family @@ -4383,6 +4384,7 @@ text = comment token = self.tokens['edit'] result = None + file_page_title = filepage.title(withNamespace=False) if source_filename: # upload local file # make sure file actually exists @@ -4392,20 +4394,22 @@ additional_parameters = {} throttle = True filesize = os.path.getsize(source_filename) - if (chunk_size > 0 and chunk_size < filesize and - LV(self.version()) >= LV('1.20')): - offset = 0 - file_key = None - with open(source_filename, 'rb') as f: + chunked_upload = (chunk_size > 0 and chunk_size < filesize and + LV(self.version()) >= LV('1.20')) + with open(source_filename, 'rb') as f: + if chunked_upload: + offset = 0 + file_key = None while True: f.seek(offset) chunk = f.read(chunk_size) - file_page_title = filepage.title(withNamespace=False) req = api.Request(site=self, action='upload', token=token, stash='1', offset=offset, filesize=filesize, filename=file_page_title, mime_params={}, throttle=throttle) - req.mime_params['chunk'] = (chunk, None, {'filename': file_page_title}) + req.mime_params['chunk'] = (chunk, + ("application", "octet-stream"), + {'filename': file_page_title}) if file_key: req['filekey'] = file_key try: @@ -4432,10 +4436,19 @@ if data['result'] != 'Continue': # finished additional_parameters['filekey'] = file_key break - else: - additional_parameters = {'file': source_filename, 'mime': True} + else: # not chunked upload + file_contents = f.read() + filetype = (mimetypes.guess_type(source_filename)[0] + or 'application/octet-stream') + additional_parameters = { + 'mime_params': { + 'file': (file_contents, + filetype.split('/'), + {'filename': file_page_title}) + } + } req = api.Request(site=self, action="upload", token=token, - filename=filepage.title(withNamespace=False), + filename=file_page_title, comment=comment, text=text, throttle=throttle, **additional_parameters) else: @@ -4445,7 +4458,7 @@ "User '%s' is not authorized to upload by URL on site %s." % (self.user(), self)) req = api.Request(site=self, action="upload", token=token, - filename=filepage.title(withNamespace=False), + filename=file_page_title, url=source_url, comment=comment, text=text) if not result: if watch: -- To view, visit https://gerrit.wikimedia.org/r/165018 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I09abe7e8350c034f1d5a2c81d99063ab3c3f7924 Gerrit-PatchSet: 5 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] Use page title for Content-disposition filename - change (pywikibot/core)
by jenkins-bot (Code Review) 13 Oct '14

13 Oct '14

jenkins-bot has submitted this change and it was merged. Change subject: Use page title for Content-disposition filename ...................................................................... Use page title for Content-disposition filename Pywikibot has sent the local filename as the Content-disposition filename since it was first implemented in Pywikibot 2.0: f08b87dd56d4096e82eea6b6f0c35e260b2038f2 This is unnecessary, and leaks the local path of the file. Pywikipedia sends the filename returned by process_filename(), which doesnt include the path. Change-Id: Ib751ee3f4074a60f3b53b0afe3cc2dfc3e17b2f7 --- M pywikibot/data/api.py 1 file changed, 1 insertion(+), 1 deletion(-) Approvals: John Vandenberg: Looks good to me, but someone else must approve XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index 2cac9a3..e5cb9f8 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -494,7 +494,7 @@ file_content = file(local_filename, "rb").read() submsg = Request._generate_MIME_part( key, file_content, filetype.split('/'), - {'filename': local_filename}) + {'filename': self._params['filename'][0]}) else: submsg = Request._generate_MIME_part( key, value, None, None) -- To view, visit https://gerrit.wikimedia.org/r/165588 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ib751ee3f4074a60f3b53b0afe3cc2dfc3e17b2f7 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] Deprecation decorator tests - change (pywikibot/core)
by jenkins-bot (Code Review) 13 Oct '14

13 Oct '14

jenkins-bot has submitted this change and it was merged. Change subject: Deprecation decorator tests ...................................................................... Deprecation decorator tests Add tests for decorators @deprecated and @deprecate_arg Change-Id: I0cb8edd929b7c335cfed1e82a747f4c6c6c726cb --- M tests/__init__.py M tests/aspects.py A tests/deprecation_tests.py M tox.ini 4 files changed, 456 insertions(+), 0 deletions(-) Approvals: John Vandenberg: Looks good to me, but someone else must approve XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/__init__.py b/tests/__init__.py index 9331544..e2bdd7a 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -51,6 +51,7 @@ _data_dir = os.path.join(_tests_dir, 'data') library_test_modules = [ + 'deprecation', 'date', 'mediawikiversion', 'ipregex', diff --git a/tests/aspects.py b/tests/aspects.py index cbd870c..08db133 100644 --- a/tests/aspects.py +++ b/tests/aspects.py @@ -657,3 +657,44 @@ pwb = True spawn = True + + +class DeprecationTestCase(TestCase): + + """Test cases for deprecation function in the tools module.""" + + deprecation_messages = [] + + @staticmethod + def _record_messages(msg): + DeprecationTestCase.deprecation_messages.append(msg) + + @staticmethod + def _reset_messages(): + DeprecationTestCase.deprecation_messages = [] + + def assertDeprecation(self, msg): + self.assertIn(msg, DeprecationTestCase.deprecation_messages) + + def assertNoDeprecation(self, msg=None): + if msg: + self.assertNotIn(msg, DeprecationTestCase.deprecation_messages) + else: + self.assertEqual([], DeprecationTestCase.deprecation_messages) + + def setUp(self): + self.tools_warning = pywikibot.tools.warning + self.tools_debug = pywikibot.tools.debug + + pywikibot.tools.warning = DeprecationTestCase._record_messages + pywikibot.tools.debug = DeprecationTestCase._record_messages + + super(DeprecationTestCase, self).setUp() + + DeprecationTestCase._reset_messages() + + def tearDown(self): + pywikibot.tools.warning = self.tools_warning + pywikibot.tools.debug = self.tools_warning + + super(DeprecationTestCase, self).tearDown() diff --git a/tests/deprecation_tests.py b/tests/deprecation_tests.py new file mode 100644 index 0000000..6880bdd --- /dev/null +++ b/tests/deprecation_tests.py @@ -0,0 +1,413 @@ +# -*- coding: utf-8 -*- +"""Tests for deprecation tools.""" +# +# (C) Pywikibot team, 2014 +# +# Distributed under the terms of the MIT license. +# +__version__ = '$Id$' + +from pywikibot.tools import deprecated, deprecate_arg +from tests.aspects import unittest, DeprecationTestCase + + +@deprecated() +def deprecated_func(foo=None): + """Deprecated function.""" + return foo + + +@deprecated +def deprecated_func2(foo=None): + """Deprecated function.""" + return foo + + +@deprecated() +def deprecated_func_bad_args(self): + """Deprecated function with arg 'self'.""" + return self + + +@deprecate_arg('bah', 'foo') +def deprecated_func_arg(foo=None): + """Deprecated function with arg 'self'.""" + return foo + + +class DeprecatedMethodClass(object): + + """Class with methods deprecated.""" + + @classmethod + @deprecated() + def class_method(cls, foo=None): + return foo + + @staticmethod + @deprecated() + def static_method(foo=None): + return foo + + @deprecated() + def instance_method(self, foo=None): + self.foo = foo + return foo + + @deprecated + def instance_method2(self, foo=None): + self.foo = foo + return foo + + def undecorated_method(self, foo=None): + return foo + + @deprecate_arg('bah', 'foo') + def deprecated_instance_method_arg(self, foo=None): + self.foo = foo + return foo + + @deprecate_arg('bah', 'foo') + @deprecate_arg('bah2', 'foo2') + def deprecated_instance_method_args(self, foo, foo2): + self.foo = foo + self.foo2 = foo2 + return (foo, foo2) + + @deprecated() + @deprecate_arg('bah', 'foo') + def deprecated_instance_method_and_arg(self, foo): + self.foo = foo + return foo + + @deprecate_arg('bah', 'foo') + @deprecated() + def deprecated_instance_method_and_arg2(self, foo): + self.foo = foo + return foo + + +@deprecated() +class DeprecatedClassNoInit(object): + + """Deprecated class.""" + + pass + + +@deprecated() +class DeprecatedClass(object): + + """Deprecated class.""" + + def __init__(self, foo=None): + self.foo = foo + + +class DeprecatorTestCase(DeprecationTestCase): + + """Test cases for deprecation tools.""" + + net = False + + @unittest.expectedFailure + def test_deprecated_function_zero_arg(self): + """Test @deprecated with functions, with zero arguments.""" + rv = deprecated_func() + self.assertEqual(rv, None) + self.assertDeprecation('deprecated_func is DEPRECATED.') + + def test_deprecated_function(self): + """Test @deprecated with functions.""" + rv = deprecated_func('a') + self.assertEqual(rv, 'a') + self.assertDeprecation('str.deprecated_func is DEPRECATED.') + + DeprecatorTestCase._reset_messages() + + rv = deprecated_func(1) + self.assertEqual(rv, 1) + self.assertDeprecation('int.deprecated_func is DEPRECATED.') + + @unittest.expectedFailure + def test_deprecated_function2(self): + """Test @deprecated with functions.""" + rv = deprecated_func2('a') + self.assertEqual(rv, 'a') + self.assertDeprecation('str.deprecated_func is DEPRECATED.') + + DeprecatorTestCase._reset_messages() + + rv = deprecated_func2(1) + self.assertEqual(rv, 1) + self.assertDeprecation('int.deprecated_func is DEPRECATED.') + + def test_deprecated_function_bad_args(self): + """Test weakness in @deprecated.""" + rv = deprecated_func_bad_args(None) + self.assertEqual(rv, None) + self.assertDeprecation('NoneType.deprecated_func_bad_args is DEPRECATED.') + + DeprecatorTestCase._reset_messages() + + rv = deprecated_func_bad_args('a') + self.assertEqual(rv, 'a') + self.assertDeprecation('str.deprecated_func_bad_args is DEPRECATED.') + + DeprecatorTestCase._reset_messages() + + rv = deprecated_func_bad_args(1) + self.assertEqual(rv, 1) + self.assertDeprecation('int.deprecated_func_bad_args is DEPRECATED.') + + def test_deprecated_instance_method(self): + f = DeprecatedMethodClass() + + rv = f.instance_method() + self.assertEqual(rv, None) + self.assertEqual(f.foo, None) + self.assertDeprecation('DeprecatedMethodClass.instance_method is DEPRECATED.') + + DeprecatorTestCase._reset_messages() + + rv = f.instance_method('a') + self.assertEqual(rv, 'a') + self.assertEqual(f.foo, 'a') + self.assertDeprecation('DeprecatedMethodClass.instance_method is DEPRECATED.') + + DeprecatorTestCase._reset_messages() + + rv = f.instance_method(1) + self.assertEqual(rv, 1) + self.assertEqual(f.foo, 1) + self.assertDeprecation('DeprecatedMethodClass.instance_method is DEPRECATED.') + + @unittest.expectedFailure + def test_deprecated_instance_method2(self): + f = DeprecatedMethodClass() + + rv = f.instance_method2() + self.assertEqual(rv, None) + self.assertEqual(f.foo, None) + self.assertDeprecation('DeprecatedMethodClass.instance_method2 is DEPRECATED.') + + def test_deprecated_class_method(self): + """Test @deprecated with class methods.""" + rv = DeprecatedMethodClass.class_method() + self.assertEqual(rv, None) + self.assertDeprecation('type.class_method is DEPRECATED.') + + DeprecatorTestCase._reset_messages() + + rv = DeprecatedMethodClass.class_method('a') + self.assertEqual(rv, 'a') + self.assertDeprecation('type.class_method is DEPRECATED.') + + DeprecatorTestCase._reset_messages() + + rv = DeprecatedMethodClass.class_method(1) + self.assertEqual(rv, 1) + self.assertDeprecation('type.class_method is DEPRECATED.') + + @unittest.expectedFailure + def test_deprecated_static_method_zero_args(self): + """Test @deprecated with static methods, with zero arguments.""" + rv = DeprecatedMethodClass.static_method() + # raises: IndexError: tuple index out of range + # because args[0] is used even if args is empty + self.assertEqual(rv, None) + self.assertDeprecation('DeprecatedMethodClass.static_method is DEPRECATED.') + + def test_deprecated_static_method(self): + """Test @deprecated with static methods.""" + rv = DeprecatedMethodClass.static_method('a') + self.assertEqual(rv, 'a') + self.assertDeprecation('str.static_method is DEPRECATED.') + + DeprecatorTestCase._reset_messages() + + rv = DeprecatedMethodClass.static_method(1) + self.assertEqual(rv, 1) + self.assertDeprecation('int.static_method is DEPRECATED.') + + @unittest.expectedFailure + def test_deprecate_class_zero_arg(self): + """Test @deprecated with classes, without arguments.""" + df = DeprecatedClassNoInit() + # raises: IndexError: tuple index out of range + # because args[0] is used even if args is empty + self.assertEqual(df.__doc__, 'Deprecated class.') + self.assertDeprecation('DeprecatedClassNoInit is DEPRECATED.') + + DeprecatorTestCase._reset_messages() + + df = DeprecatedClass() + self.assertEqual(df.foo, None) + self.assertDeprecation('DeprecatedClass is DEPRECATED.') + + def test_deprecate_class(self): + """Test @deprecated with classes.""" + df = DeprecatedClass('a') + self.assertEqual(df.foo, 'a') + self.assertDeprecation('str.DeprecatedClass is DEPRECATED.') + + def test_deprecated_function_arg(self): + """Test @deprecate_arg with function arguments.""" + rv = deprecated_func_arg() + self.assertEqual(rv, None) + self.assertNoDeprecation() + + rv = deprecated_func_arg('a') + self.assertEqual(rv, 'a') + self.assertNoDeprecation() + + rv = deprecated_func_arg(bah='b') + self.assertEqual(rv, 'b') + self.assertDeprecation('bah argument of deprecated_func_arg is deprecated; use foo instead.') + + DeprecatorTestCase._reset_messages() + + rv = deprecated_func_arg(foo=1) + self.assertEqual(rv, 1) + self.assertNoDeprecation() + + def test_deprecated_instance_method_zero_arg(self): + """Test @deprecate_arg with classes, without arguments.""" + f = DeprecatedMethodClass() + + rv = f.deprecated_instance_method_arg() + self.assertEqual(rv, None) + self.assertEqual(f.foo, None) + self.assertNoDeprecation() + + def test_deprecated_instance_method_arg(self): + """Test @deprecate_arg with instance methods.""" + f = DeprecatedMethodClass() + + rv = f.deprecated_instance_method_arg('a') + self.assertEqual(rv, 'a') + self.assertEqual(f.foo, 'a') + self.assertNoDeprecation() + + rv = f.deprecated_instance_method_arg(bah='b') + self.assertEqual(rv, 'b') + self.assertEqual(f.foo, 'b') + self.assertDeprecation( + 'bah argument of deprecated_instance_method_arg is deprecated; use foo instead.') + + DeprecatorTestCase._reset_messages() + + rv = f.deprecated_instance_method_arg(foo=1) + self.assertEqual(rv, 1) + self.assertEqual(f.foo, 1) + self.assertNoDeprecation() + + def test_deprecated_instance_method_args(self): + """Test @deprecate_arg with instance methods and two args.""" + f = DeprecatedMethodClass() + + rv = f.deprecated_instance_method_args('a', 'b') + self.assertEqual(rv, ('a', 'b')) + self.assertNoDeprecation() + + rv = f.deprecated_instance_method_args(bah='b', bah2='c') + self.assertEqual(rv, ('b', 'c')) + self.assertDeprecation( + 'bah argument of deprecated_instance_method_args is deprecated; use foo instead.') + self.assertDeprecation( + 'bah2 argument of deprecated_instance_method_args is deprecated; use foo2 instead.') + + DeprecatorTestCase._reset_messages() + + rv = f.deprecated_instance_method_args(foo='b', bah2='c') + self.assertEqual(rv, ('b', 'c')) + self.assertNoDeprecation( + 'bah argument of deprecated_instance_method_args is deprecated; use foo instead.') + self.assertDeprecation( + 'bah2 argument of deprecated_instance_method_args is deprecated; use foo2 instead.') + + DeprecatorTestCase._reset_messages() + + rv = f.deprecated_instance_method_args(foo2='c', bah='b') + self.assertEqual(rv, ('b', 'c')) + self.assertDeprecation( + 'bah argument of deprecated_instance_method_args is deprecated; use foo instead.') + self.assertNoDeprecation( + 'bah2 argument of deprecated_instance_method_args is deprecated; use foo2 instead.') + + DeprecatorTestCase._reset_messages() + + rv = f.deprecated_instance_method_args(foo=1, foo2=2) + self.assertEqual(rv, (1, 2)) + self.assertNoDeprecation() + + def test_deprecated_instance_method_and_arg(self): + """Test @deprecate_arg and @deprecated with instance methods.""" + f = DeprecatedMethodClass() + + rv = f.deprecated_instance_method_and_arg('a') + self.assertEqual(rv, 'a') + self.assertEqual(f.foo, 'a') + self.assertDeprecation( + 'DeprecatedMethodClass.deprecated_instance_method_and_arg is DEPRECATED.') + self.assertNoDeprecation( + 'bah argument of deprecated_instance_method_and_arg is deprecated; use foo instead.') + + DeprecatorTestCase._reset_messages() + + rv = f.deprecated_instance_method_and_arg(bah='b') + self.assertEqual(rv, 'b') + self.assertEqual(f.foo, 'b') + self.assertDeprecation( + 'DeprecatedMethodClass.deprecated_instance_method_and_arg is DEPRECATED.') + self.assertDeprecation( + 'bah argument of deprecated_instance_method_and_arg is deprecated; use foo instead.') + + DeprecatorTestCase._reset_messages() + + rv = f.deprecated_instance_method_and_arg(foo=1) + self.assertEqual(rv, 1) + self.assertEqual(f.foo, 1) + self.assertDeprecation( + 'DeprecatedMethodClass.deprecated_instance_method_and_arg is DEPRECATED.') + self.assertNoDeprecation( + 'bah argument of deprecated_instance_method_and_arg is deprecated; use foo instead.') + + def test_deprecated_instance_method_and_arg2(self): + """Test @deprecate_arg and @deprecated with instance methods.""" + f = DeprecatedMethodClass() + + rv = f.deprecated_instance_method_and_arg2('a') + self.assertEqual(rv, 'a') + self.assertEqual(f.foo, 'a') + self.assertDeprecation( + 'DeprecatedMethodClass.deprecated_instance_method_and_arg2 is DEPRECATED.') + self.assertNoDeprecation( + 'bah argument of deprecated_instance_method_and_arg2 is deprecated; use foo instead.') + + DeprecatorTestCase._reset_messages() + + rv = f.deprecated_instance_method_and_arg2(bah='b') + self.assertEqual(rv, 'b') + self.assertEqual(f.foo, 'b') + self.assertDeprecation( + 'DeprecatedMethodClass.deprecated_instance_method_and_arg2 is DEPRECATED.') + self.assertDeprecation( + 'bah argument of deprecated_instance_method_and_arg2 is deprecated; use foo instead.') + + DeprecatorTestCase._reset_messages() + + rv = f.deprecated_instance_method_and_arg2(foo=1) + self.assertEqual(rv, 1) + self.assertEqual(f.foo, 1) + self.assertDeprecation( + 'DeprecatedMethodClass.deprecated_instance_method_and_arg2 is DEPRECATED.') + self.assertNoDeprecation( + 'bah argument of deprecated_instance_method_and_arg2 is deprecated; use foo instead.') + + +if __name__ == '__main__': + try: + unittest.main() + except SystemExit: + pass diff --git a/tox.ini b/tox.ini index ed4e709..cd9a7e0 100644 --- a/tox.ini +++ b/tox.ini @@ -57,6 +57,7 @@ ./scripts/illustrate_wikidata.py \ ./scripts/newitem.py \ ./tests/aspects.py \ + ./tests/deprecation_tests.py \ ./tests/api_tests.py \ ./tests/dry_api_tests.py \ ./scripts/maintenance/cache.py \ -- To view, visit https://gerrit.wikimedia.org/r/166171 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I0cb8edd929b7c335cfed1e82a747f4c6c6c726cb Gerrit-PatchSet: 6 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: Nullzero <nullzero.free(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] Split testImageusage into four tests - change (pywikibot/core)
by jenkins-bot (Code Review) 13 Oct '14

13 Oct '14

jenkins-bot has submitted this change and it was merged. Change subject: Split testImageusage into four tests ...................................................................... Split testImageusage into four tests There is a reoccurring problem with this test method on English Wikipedia. As the test currently has four different sets of tests, it is not 100% reliable to determine which queries are causing the problem. Output message regarding which image is being used in these tests. Bug: 71971 Change-Id: I8443bc0f55ff77b8fc266f55c2bfbf32940be892 --- M tests/site_tests.py 1 file changed, 45 insertions(+), 4 deletions(-) Approvals: John Vandenberg: Looks good to me, but someone else must approve Mpaa: Looks good to me, approved jenkins-bot: Verified diff --git a/tests/site_tests.py b/tests/site_tests.py index c9b99bf..3f95d9b 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -591,11 +591,27 @@ self.assertIsInstance(link, pywikibot.Page) self.assertIn(link.namespace(), (2, 3)) - def testImageusage(self): - """Test the site.imageusage() method""" + +class TestImageUsage(DefaultSiteTestCase): + + """Test cases for Site.imageusage method.""" + + cached = True + + @property + def imagepage(self): + """Find an image which is used on a page. + + If there are no images included in pages it'll skip all tests. + + Note: This is not implemented as setUpClass which would be invoked + while initialising all tests, to reduce chance of an error preventing + all tests from running. + """ + if hasattr(self.__class__, '_image_page'): + return self.__class__._image_page + mysite = self.get_site() - # Find a page which uses an image, so we can test imageusage - # If there are no images included in pages it'll skip this test for page in mysite.allpages(filterredir=False): try: imagepage = next(iter(page.imagelinks())) # 1st image of page @@ -606,16 +622,41 @@ else: raise unittest.SkipTest("No images on site {0!r}".format(mysite)) + pywikibot.output(u'site_tests.TestImageUsage found %s on %s' + % (imagepage, page)) + + self.__class__._image_page = imagepage + return imagepage + + def test_image_usage(self): + """Test the site.imageusage() method.""" + mysite = self.get_site() + imagepage = self.imagepage iu = list(mysite.imageusage(imagepage, total=10)) self.assertLessEqual(len(iu), 10) self.assertTrue(all(isinstance(link, pywikibot.Page) for link in iu)) + + def test_image_usage_in_namespaces(self): + """Test the site.imageusage() method with namespaces.""" + mysite = self.get_site() + imagepage = self.imagepage for using in mysite.imageusage(imagepage, namespaces=[3, 4], total=5): self.assertIsInstance(using, pywikibot.Page) self.assertIn(imagepage, list(using.imagelinks())) + + def test_image_usage_in_redirects(self): + """Test the site.imageusage() method on redirects only.""" + mysite = self.get_site() + imagepage = self.imagepage for using in mysite.imageusage(imagepage, filterredir=True, total=5): self.assertIsInstance(using, pywikibot.Page) self.assertTrue(using.isRedirectPage()) + + def test_image_usage_no_redirect_filter(self): + """Test the site.imageusage() method with redirects.""" + mysite = self.get_site() + imagepage = self.imagepage for using in mysite.imageusage(imagepage, filterredir=False, total=5): self.assertIsInstance(using, pywikibot.Page) self.assertFalse(using.isRedirectPage()) -- To view, visit https://gerrit.wikimedia.org/r/166375 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I8443bc0f55ff77b8fc266f55c2bfbf32940be892 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] config2: remove useless if branch - change (pywikibot/core)
by jenkins-bot (Code Review) 13 Oct '14

13 Oct '14

jenkins-bot has submitted this change and it was merged. Change subject: config2: remove useless if branch ...................................................................... config2: remove useless if branch Removed branch since its condition is always evaluated to True because of line 731 Change-Id: I08259a96dd82275801a1ad9f48df8852cc244f26 --- M pywikibot/config2.py 1 file changed, 1 insertion(+), 2 deletions(-) Approvals: John Vandenberg: Looks good to me, approved Mpaa: Looks good to me, but someone else must approve jenkins-bot: Verified diff --git a/pywikibot/config2.py b/pywikibot/config2.py index b9a724f..523c764 100644 --- a/pywikibot/config2.py +++ b/pywikibot/config2.py @@ -732,8 +732,7 @@ _gl = list(_glv.keys()) _tp = {} for _key in _gl: - if _key[0] != '_': - _tp[_key] = type(globals()[_key]) + _tp[_key] = type(globals()[_key]) # Create an environment for user-config.py which is # a shallow copy of the core config settings, so that -- To view, visit https://gerrit.wikimedia.org/r/166514 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I08259a96dd82275801a1ad9f48df8852cc244f26 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Taueres <santoro.srg(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] config2: update variable _imported_modules - change (pywikibot/core)
by jenkins-bot (Code Review) 13 Oct '14

13 Oct '14

jenkins-bot has submitted this change and it was merged. Change subject: config2: update variable _imported_modules ...................................................................... config2: update variable _imported_modules _imported_modules is now correctly syncronized as specified in the comments Change-Id: I2f23ec8662d21c442ba80c10b76a350e73045908 --- M pywikibot/config2.py 1 file changed, 1 insertion(+), 1 deletion(-) Approvals: XZise: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/config2.py b/pywikibot/config2.py index b9a724f..1d91a4a 100644 --- a/pywikibot/config2.py +++ b/pywikibot/config2.py @@ -26,7 +26,7 @@ import sys import collections # Please keep _imported_modules in sync with the imports above -_imported_modules = ('os', 'sys') +_imported_modules = ('os', 'sys', 'collections') # IMPORTANT: # Do not change any of the variables in this file. Instead, make -- To view, visit https://gerrit.wikimedia.org/r/166513 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I2f23ec8662d21c442ba80c10b76a350e73045908 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Taueres <santoro.srg(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] [FIX] Link: Use unicode in error messages - change (pywikibot/core)
by jenkins-bot (Code Review) 13 Oct '14

13 Oct '14

jenkins-bot has submitted this change and it was merged. Change subject: [FIX] Link: Use unicode in error messages ...................................................................... [FIX] Link: Use unicode in error messages Change-Id: I8b0aafe9824a519fffc44a6fb173e1924a994c17 --- M pywikibot/page.py 1 file changed, 6 insertions(+), 6 deletions(-) Approvals: Mpaa: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/page.py b/pywikibot/page.py index a4ce5cb..6bf04b0 100644 --- a/pywikibot/page.py +++ b/pywikibot/page.py @@ -4052,7 +4052,7 @@ # 'namespace:' is not a valid title if not t: raise pywikibot.InvalidTitle( - "'{0}' has no title.".format(self._text)) + u"'{0}' has no title.".format(self._text)) self._namespace = ns break try: @@ -4061,7 +4061,7 @@ break # text before : doesn't match any known prefix except NoSuchSite: raise pywikibot.Error( - '{0} is not a local page on {1}, and the interwiki prefix ' + u'{0} is not a local page on {1}, and the interwiki prefix ' '{2} is not supported by PyWikiBot!'.format( self._text, self._site, prefix)) else: @@ -4069,7 +4069,7 @@ if first_other_site: if not self._site.local_interwiki(prefix): raise pywikibot.InvalidTitle( - '{0} links to a non local site {1} via an ' + u'{0} links to a non local site {1} via an ' 'interwiki link to {2}.'.format( self._text, newsite, first_other_site)) elif newsite != self._source: @@ -4102,15 +4102,15 @@ or t.endswith(u"/..") ): raise pywikibot.InvalidTitle( - "(contains . / combinations): '%s'" + u"(contains . / combinations): '%s'" % self._text) # Magic tilde sequences? Nu-uh! if u"~~~" in t: - raise pywikibot.InvalidTitle("(contains ~~~): '%s'" % self._text) + raise pywikibot.InvalidTitle(u"(contains ~~~): '%s'" % self._text) if self._namespace != -1 and len(t) > 255: - raise pywikibot.InvalidTitle("(over 255 bytes): '%s'" % t) + raise pywikibot.InvalidTitle(u"(over 255 bytes): '%s'" % t) # "empty" local links can only be self-links # with a fragment identifier. -- To view, visit https://gerrit.wikimedia.org/r/166426 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I8b0aafe9824a519fffc44a6fb173e1924a994c17 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] Remove unnecessary unicode from i18n docstring - change (pywikibot/core)
by XZise (Code Review) 13 Oct '14

13 Oct '14

XZise has submitted this change and it was merged. Change subject: Remove unnecessary unicode from i18n docstring ...................................................................... Remove unnecessary unicode from i18n docstring The unicode in a r"""..""" was the last pep257 issue. Change-Id: I10b8587b2c6d9a42e46e63c9f0fdb9747f5d6040 --- M pywikibot/i18n.py M tox.ini 2 files changed, 1 insertion(+), 3 deletions(-) Approvals: XZise: Looks good to me, approved diff --git a/pywikibot/i18n.py b/pywikibot/i18n.py index e10ccbf..33ad626 100644 --- a/pywikibot/i18n.py +++ b/pywikibot/i18n.py @@ -418,9 +418,6 @@ Examples: If we had a test dictionary in test.py like msg = { - 'de': { - 'test-changing': u'Bot: Ändere %(num)d {{PLURAL:num|Seite|Seiten}}.', - }, 'en': { # number value as format sting is allowed 'test-changing': u'Bot: Changing %(num)s {{PLURAL:%(num)d|page|pages}}.', diff --git a/tox.ini b/tox.ini index ed4e709..2d2b443 100644 --- a/tox.ini +++ b/tox.ini @@ -36,6 +36,7 @@ ./pywikibot/editor.py \ ./pywikibot/family.py \ ./pywikibot/fixes.py \ + ./pywikibot/i18n.py \ ./pywikibot/interwiki_graph.py \ ./pywikibot/page.py \ ./pywikibot/plural.py \ -- To view, visit https://gerrit.wikimedia.org/r/166422 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I10b8587b2c6d9a42e46e63c9f0fdb9747f5d6040 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Siebrand <siebrand(a)kitano.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] [FIX] Allow dynamic interwiki links - change (pywikibot/core)
by jenkins-bot (Code Review) 13 Oct '14

13 Oct '14

jenkins-bot has submitted this change and it was merged. Change subject: [FIX] Allow dynamic interwiki links ...................................................................... [FIX] Allow dynamic interwiki links This uses the 'interwikimap' siteinfo to determine the Site an interwiki prefix is redirecting to. A Family supports 'from_url' which returns the 'code' (an entry in 'langs') to which the URL belongs to. If no code of that family does support that URL 'None' is returned. Because both the test and wikipedia family support the URL test.wikipedia.org, the test family returns always None so it chooses the wikipedia family. This is then consistent with the test wikidata instance which is in the wikidata family and not in a separate test family. To get the family for a specific URL it might need to iterate through all families, which are loaded in the process. Each site has a local cache which holds the Site represented by a specific interwiki prefix. There is also a global cache which stores the family name and code for each URL so that later searches for the URL don't require to iterate through all families. This also adds an parameter "url" to pywikibot.Site function which can only be set when neither "code" nor "fam" are set. The code used in 'nice_get_address' should be returned when the url is parsed by 'from_url'. This also fixes all link tests so that the 'show_failures' features could be removed. It also changes the exception from Error in InvalidTitle, when there is no title given and it has a namespace or no interwiki link. It also only throws an InvalidTitle if an interwiki link links to a non local page via a local interwiki link. Previously all interwiki links which referred to it's own site wouldn't be allowed. Change-Id: Iec4f32fdefde15b2330dbb191fb3bbb0d04803e2 --- M pywikibot/__init__.py M pywikibot/families/test_family.py M pywikibot/family.py M pywikibot/page.py M pywikibot/site.py M tests/interwiki_link_tests.py M tests/link_tests.py 7 files changed, 354 insertions(+), 288 deletions(-) Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/__init__.py b/pywikibot/__init__.py index 5257f65..2e569c0 100644 --- a/pywikibot/__init__.py +++ b/pywikibot/__init__.py @@ -459,9 +459,10 @@ _sites = {} +_url_cache = {} # The code/fam pair for each URL -def Site(code=None, fam=None, user=None, sysop=None, interface=None): +def Site(code=None, fam=None, user=None, sysop=None, interface=None, url=None): """A factory method to obtain a Site object. Site objects are cached and reused by this method. @@ -479,12 +480,40 @@ @type sysop: unicode @param interface: site interface (override config.site_interface) @type interface: string + @param url: Instead of code and fam, does try to get a Site based on the + URL. Still requires that the family supporting that URL exists. + @type url: string """ + # Either code and fam or only url + assert(not url or (not code and not fam)) _logger = "wiki" - # Fallback to config defaults - code = code or config.mylang - fam = fam or config.family + if url: + if url in _url_cache: + cached = _url_cache[url] + if cached: + code = cached[0] + fam = cached[1] + else: + raise Error("Unknown URL '{0}'.".format(url)) + else: + # Iterate through all families and look, which does apply to + # the given URL + for fam in config.family_files: + family = pywikibot.family.Family.load(fam) + code = family.from_url(url) + if code: + _url_cache[url] = (code, fam) + break + else: + _url_cache[url] = None + # TODO: As soon as AutoFamily is ready, try and use an + # AutoFamily + raise Error("Unknown URL '{0}'.".format(url)) + else: + # Fallback to config defaults + code = code or config.mylang + fam = fam or config.family interface = interface or config.site_interface # config.usernames is initialised with a dict for each family name diff --git a/pywikibot/families/test_family.py b/pywikibot/families/test_family.py index 27d5f4b..c2cc459 100644 --- a/pywikibot/families/test_family.py +++ b/pywikibot/families/test_family.py @@ -13,3 +13,6 @@ self.langs = { 'test': 'test.wikipedia.org', } + + def from_url(self, url): + return None # Don't accept this, but 'test' of 'wikipedia' diff --git a/pywikibot/family.py b/pywikibot/family.py index 1bfd558..6765176 100644 --- a/pywikibot/family.py +++ b/pywikibot/family.py @@ -1035,6 +1035,64 @@ def nice_get_address(self, code, title): return '%s%s' % (self.nicepath(code), title) + def _get_path_regex(self): + """ + Return a regex matching the path after the domain. + + It is using L{Family.path} and L{Family.nicepath} with code set to + 'None'. If that returns a KeyError (L{Family.scriptpath} probably + using the C{langs} dictionary) it retries it with the key from + L{Family.langs} if it only contains one entry and throws an Error + otherwise. In that case the Family instance should overwrite this + method or supply code independent methods. + + @raise Error: If it's not possible to automatically get a code + independent regex. + """ + def _get_coded_path_regex(code): + return ('(?:' + re.escape(self.path(code) + '/') + '|' + + re.escape(self.nicepath(code)) + ')') + try: + return _get_coded_path_regex(None) + except KeyError: + # Probably automatically generated family + if len(self.langs) == 1: + return _get_coded_path_regex(next(iter(self.langs.keys()))) + else: + raise Error('Pywikibot is unable to generate an automatic ' + 'path regex for the family {0}. It is recommended ' + 'to overwrite "_get_path_regex" in that ' + 'family.'.format(self.name)) + + def from_url(self, url): + """ + Return whether this family matches the given url. + + The protocol must match, if it is present in the URL. It must match + URLs generated via C{self.langs} and L{Family.nice_get_address} or + L{Family.path}. + + It uses L{Family._get_path_regex} to generate a regex defining the path + after the domain. + + @return: The language code of the url. None if that url is not from + this family. + @rtype: str or None + """ + url_match = re.match(r'(?:(https?)://|//)?(.*){0}' + '\$1'.format(self._get_path_regex()), url) + if not url_match: + return None + for code, domain in self.langs.items(): + if domain == url_match.group(2): + break + else: + return None + if url_match.group(1) and url_match.group(1) != self.protocol(code): + return None + else: + return code + def dbName(self, code): # returns the name of the MySQL database return '%s%s' % (code, self.name) diff --git a/pywikibot/page.py b/pywikibot/page.py index 70f8dd1..d4f4c08 100644 --- a/pywikibot/page.py +++ b/pywikibot/page.py @@ -44,7 +44,7 @@ from pywikibot import config from pywikibot.family import Family from pywikibot.site import Namespace -from pywikibot.exceptions import AutoblockUser, UserActionRefuse +from pywikibot.exceptions import AutoblockUser, UserActionRefuse, NoSuchSite from pywikibot.tools import ComparableMixin, deprecated, deprecate_arg from pywikibot import textlib @@ -4028,7 +4028,7 @@ # This code was adapted from Title.php : secureAndSplit() # - firstPass = True + first_other_site = None while u":" in t: # Initial colon indicates main namespace rather than default if t.startswith(u":"): @@ -4038,50 +4038,37 @@ t = t.lstrip(u":").lstrip(u" ") continue - fam = self._site.family prefix = t[:t.index(u":")].lower() ns = self._site.ns_index(prefix) if ns: # Ordinary namespace t = t[t.index(u":"):].lstrip(u":").lstrip(u" ") + # 'namespace:' is not a valid title + if not t: + raise pywikibot.InvalidTitle( + "'{0}' has no title.".format(self._text)) self._namespace = ns break - if prefix in list(fam.langs.keys())\ - or prefix in fam.get_known_families(site=self._site): - # looks like an interwiki link - if not firstPass: - # Can't make a local interwiki link to an interwiki link. - raise pywikibot.Error( - "Improperly formatted interwiki link '%s'" - % self._text) - t = t[t.index(u":"):].lstrip(u":").lstrip(u" ") - if prefix in list(fam.langs.keys()): - newsite = pywikibot.Site(prefix, fam) - else: - otherlang = self._site.code - familyName = fam.get_known_families(site=self._site)[prefix] - if familyName in ['commons', 'meta']: - otherlang = familyName - try: - newsite = pywikibot.Site(otherlang, familyName) - except ValueError: - raise pywikibot.Error( - """\ -%s is not a local page on %s, and the %s family is -not supported by PyWikiBot!""" - % (self._text, self._site, familyName)) - - # Redundant interwiki prefix to the local wiki - if newsite == self._site: - if not t: - # Can't have an empty self-link - raise pywikibot.InvalidTitle( - "Invalid link title: '%s'" % self._text) - firstPass = False - continue - self._site = newsite + try: + newsite = self._site.interwiki(prefix) + except KeyError: + break # text before : doesn't match any known prefix + except NoSuchSite: + raise pywikibot.Error( + '{0} is not a local page on {1}, and the interwiki prefix ' + '{2} is not supported by PyWikiBot!'.format( + self._text, self._site, prefix)) else: - break # text before : doesn't match any known prefix + t = t[t.index(u":"):].lstrip(u":").lstrip(u" ") + if first_other_site: + if not self._site.local_interwiki(prefix): + raise pywikibot.InvalidTitle( + '{0} links to a non local site {1} via an ' + 'interwiki link to {2}.'.format( + self._text, newsite, first_other_site)) + elif newsite != self._source: + first_other_site = newsite + self._site = newsite if u"#" in t: t, sec = t.split(u'#', 1) @@ -4119,6 +4106,12 @@ if self._namespace != -1 and len(t) > 255: raise pywikibot.InvalidTitle("(over 255 bytes): '%s'" % t) + # "empty" local links can only be self-links + # with a fragment identifier. + if not self._text.strip(): + raise pywikibot.InvalidTitle("The link does not contain a page " + "title") + if hasattr(self._site.namespaces()[self._namespace], 'case'): case = self._site.namespaces()[self._namespace].case else: @@ -4126,13 +4119,6 @@ if case == 'first-letter': t = t[:1].upper() + t[1:] - - # Can't make a link to a namespace alone... - # "empty" local links can only be self-links - # with a fragment identifier. - if not t and self._site == self._source and self._namespace != 0: - raise pywikibot.Error("Invalid link (no page title): '%s'" - % self._text) self._title = t diff --git a/pywikibot/site.py b/pywikibot/site.py index 17c47aa..09a915b 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -542,6 +542,55 @@ return [lang for lang in self.languages() if lang[:1].upper() + lang[1:] not in nsnames] + def interwiki(self, prefix): + """ + Return the site for a corresponding interwiki prefix. + + @raise NoSuchSite: if the url given in the interwiki table doesn't + match any of the existing families. + @raise KeyError: if the prefix is not an interwiki prefix. + """ + # _iw_sites is a local cache to return a APISite instance depending + # on the interwiki prefix of that site + if not hasattr(self, '_iw_sites'): + self._iw_sites = {} + if prefix in self._iw_sites: + site = self._iw_sites[prefix] + else: + for interwiki in self.siteinfo['interwikimap']: + if interwiki['prefix'] == prefix: + break + else: + raise KeyError( + "'{0}' is not an interwiki prefix.".format(prefix)) + try: + site = (pywikibot.Site(url=interwiki['url']), + 'local' in interwiki) + except Error: + site = (None, False) + self._iw_sites[prefix] = site + if site[0]: + return site[0] + else: + raise NoSuchSite( + "No family/site found for prefix '{0}'".format(prefix)) + + def local_interwiki(self, prefix): + """ + Return whether the interwiki prefix is local. + + A local interwiki prefix is handled by the target site like a normal + link. So if that link also contains an interwiki link it does follow + it as long as it's a local link. + + @raise NoSuchSite: if the url given in the interwiki table doesn't + match any of the existing families. + @raise KeyError: if the prefix is not an interwiki prefix. + """ + # Request if necessary + self.interwiki(prefix) + return self._iw_sites[prefix][1] + def ns_index(self, namespace): """Given a namespace name, return its int index, or None if invalid.""" for ns in self.namespaces(): diff --git a/tests/interwiki_link_tests.py b/tests/interwiki_link_tests.py index 6b75ab7..2b34015 100644 --- a/tests/interwiki_link_tests.py +++ b/tests/interwiki_link_tests.py @@ -8,6 +8,7 @@ from pywikibot import config2 as config from pywikibot.page import Link +from pywikibot.exceptions import InvalidTitle from tests.aspects import unittest, TestCase @@ -46,6 +47,44 @@ self.assertEqual(link.namespace, 1) +class TestInterwikiLinksToNonLocalSites(TestCase): + + """Tests for interwiki links to non local sites.""" + + sites = { + 'wp': { + 'family': 'wikipedia', + 'code': 'en' + }, + 'tw': { + 'family': 'i18n', + 'code': 'i18n' + } + } + + def test_direct_non_local(self): + link = Link('translatewiki:Main Page', self.get_site('wp')) + link.parse() + self.assertEqual(link.site, self.get_site('tw')) + self.assertEqual(link.title, 'Main Page') + self.assertEqual(link.namespace, 0) + + def test_indirect_non_local(self): + link = Link('en:translatewiki:Main Page', self.get_site('wp')) + link.parse() + self.assertEqual(link.site, self.get_site('tw')) + self.assertEqual(link.title, 'Main Page') + self.assertEqual(link.namespace, 0) + + def test_via_local_non_local(self): + link = Link('de:translatewiki:Main Page', self.get_site('wp')) + self.assertRaisesRegex( + InvalidTitle, + "de:translatewiki:Main Page links to a non local site i18n:i18n " + "via an interwiki link to wikipedia:de", + link.parse) + + if __name__ == '__main__': try: unittest.main() diff --git a/tests/link_tests.py b/tests/link_tests.py index aa89306..b1ca6d0 100644 --- a/tests/link_tests.py +++ b/tests/link_tests.py @@ -6,13 +6,11 @@ # __version__ = '$Id$' -import os import pywikibot from pywikibot import config2 as config from pywikibot.page import Link +from pywikibot.exceptions import Error, InvalidTitle from tests.aspects import unittest, TestCase - -show_failures = os.environ.get('PYWIKIBOT2_TEST_SHOW_FAILURE', '0') == '1' # ---- The first set of tests are explicit links, starting with a ':'. @@ -210,50 +208,20 @@ config.mylang = 'en' config.family = 'wikisource' link = Link(':en:wikipedia:Main Page') - if show_failures: - link.parse() - else: - self.assertRaisesRegex( - pywikibot.Error, - "Improperly formatted interwiki link ':en:wikipedia:Main Page'", - link.parse) - if show_failures: - self.assertEqual(link.site, self.get_site('enwp')) - else: - self.assertEqual(link.site, self.get_site('enws')) - if show_failures: - self.assertEqual(link.title, 'Main Page') - self.assertEqual(link.namespace, 0) - else: - try: - link.title - except pywikibot.Error as e: - self.assertEqual(str(e), "Improperly formatted interwiki link ':en:wikipedia:Main Page'") + link.parse() + self.assertEqual(link.site, self.get_site('enwp')) + self.assertEqual(link.title, 'Main Page') + self.assertEqual(link.namespace, 0) def test_fully_qualified_NS1_code(self): """Test ':en:wikipedia:Main Page' on enwp is namespace 1.""" config.mylang = 'en' config.family = 'wikisource' link = Link(':en:wikipedia:Talk:Main Page') - if show_failures: - link.parse() - else: - self.assertRaisesRegex( - pywikibot.Error, - "Improperly formatted interwiki link ':en:wikipedia:Talk:Main Page'", - link.parse) - if show_failures: - self.assertEqual(link.site, self.get_site('enwp')) - else: - self.assertEqual(link.site, self.get_site('enws')) - if show_failures: - self.assertEqual(link.title, 'Main Page') - self.assertEqual(link.namespace, 1) - else: - try: - link.title - except pywikibot.Error as e: - self.assertEqual(str(e), "Improperly formatted interwiki link ':en:wikipedia:Talk:Main Page'") + link.parse() + self.assertEqual(link.site, self.get_site('enwp')) + self.assertEqual(link.title, 'Main Page') + self.assertEqual(link.namespace, 1) def test_fully_qualified_NS0_family(self): """Test ':wikipedia:en:Main Page' on enws is namespace 0.""" @@ -296,16 +264,9 @@ config.family = 'wikidata' link = Link(':en:wikipedia:Main Page') link.parse() - if show_failures: - self.assertEqual(link.site, self.get_site('enwp')) - else: - self.assertEqual(link.site, self.get_site('wikidata')) - if show_failures: - self.assertEqual(link.title, 'Main Page') - self.assertEqual(link.namespace, 4) - else: - self.assertEqual(link.title, 'En:wikipedia:Main Page') - self.assertEqual(link.namespace, 0) + self.assertEqual(link.site, self.get_site('enwp')) + self.assertEqual(link.title, 'Main Page') + self.assertEqual(link.namespace, 4) def test_fully_qualified_NS1_code(self): """Test ':en:wikipedia:Talk:Main Page' on wikidata is namespace 4.""" @@ -313,52 +274,29 @@ config.family = 'wikidata' link = Link(':en:wikipedia:Talk:Main Page') link.parse() - if show_failures: - self.assertEqual(link.site, self.get_site('enwp')) - else: - self.assertEqual(link.site, self.get_site('wikidata')) - if show_failures: - self.assertEqual(link.title, 'Talk:Main Page') - self.assertEqual(link.namespace, 4) - else: - self.assertEqual(link.title, 'En:wikipedia:Talk:Main Page') - self.assertEqual(link.namespace, 0) + self.assertEqual(link.site, self.get_site('enwp')) + self.assertEqual(link.title, 'Talk:Main Page') + self.assertEqual(link.namespace, 4) def test_fully_qualified_NS0_family(self): """Test ':wikipedia:en:Main Page' on wikidata is namespace 0.""" config.mylang = 'wikidata' config.family = 'wikidata' link = Link(':wikipedia:en:Main Page') - if show_failures: - link.parse() - self.assertEqual(link.site, self.get_site('enwp')) - self.assertEqual(link.title, 'Main Page') - self.assertEqual(link.namespace, 0) - else: - self.assertRaisesRegex( - pywikibot.NoSuchSite, - 'Language wikidata does not exist in family wikipedia', - link.parse) # very bad + link.parse() + self.assertEqual(link.site, self.get_site('enwp')) + self.assertEqual(link.title, 'Main Page') + self.assertEqual(link.namespace, 0) def test_fully_qualified_NS1_family(self): """Test ':wikipedia:en:Talk:Main Page' on wikidata is namespace 1.""" config.mylang = 'wikidata' config.family = 'wikidata' link = Link(':wikipedia:en:Talk:Main Page') - if show_failures: - link.parse() - else: - self.assertRaisesRegex( - pywikibot.NoSuchSite, - 'Language wikidata does not exist in family wikipedia', - link.parse) # very bad - if show_failures: - self.assertEqual(link.site, self.get_site('enwp')) - else: - self.assertEqual(link.site, self.get_site('wikidata')) - if show_failures: - self.assertEqual(link.title, 'Main Page') - self.assertEqual(link.namespace, 1) + link.parse() + self.assertEqual(link.site, self.get_site('enwp')) + self.assertEqual(link.title, 'Main Page') + self.assertEqual(link.namespace, 1) class TestFullyQualifiedNoLangFamilyExplicitLinkParser(TestCase): @@ -384,32 +322,20 @@ config.mylang = 'en' config.family = 'wikipedia' link = Link(':testwiki:wikidata:Q6') - if show_failures: - link.parse() - self.assertEqual(link.site, self.get_site('wikidata')) - self.assertEqual(link.title, 'Q6') - self.assertEqual(link.namespace, 0) - else: - self.assertRaisesRegex( - pywikibot.Error, - 'Family testwiki does not exist', - link.parse) # very bad + link.parse() + self.assertEqual(link.site, self.get_site('wikidata')) + self.assertEqual(link.title, 'Q6') + self.assertEqual(link.namespace, 0) def test_fully_qualified_NS1_code(self): """Test ':testwiki:wikidata:Talk:Q6' on enwp is namespace 1.""" config.mylang = 'en' config.family = 'wikipedia' link = Link(':testwiki:wikidata:Talk:Q6') - if show_failures: - link.parse() - self.assertEqual(link.site, self.get_site('wikidata')) - self.assertEqual(link.title, 'Q6') - self.assertEqual(link.namespace, 1) - else: - self.assertRaisesRegex( - pywikibot.Error, - 'Family testwiki does not exist', - link.parse) # very bad + link.parse() + self.assertEqual(link.site, self.get_site('wikidata')) + self.assertEqual(link.title, 'Q6') + self.assertEqual(link.namespace, 1) def test_fully_qualified_NS0_family(self): """Test ':wikidata:testwiki:Q6' on enwp is namespace 0.""" @@ -417,14 +343,9 @@ config.family = 'wikipedia' link = Link(':wikidata:testwiki:Q6') link.parse() - if show_failures: - self.assertEqual(link.site, self.get_site('test.wp')) - self.assertEqual(link.title, 'Q6') - self.assertEqual(link.namespace, 0) - else: - self.assertEqual(link.site, self.get_site('enwp')) - self.assertEqual(link.title, 'Wikidata:testwiki:Q6') - self.assertEqual(link.namespace, 0) + self.assertEqual(link.site, self.get_site('test.wp')) + self.assertEqual(link.title, 'Q6') + self.assertEqual(link.namespace, 0) def test_fully_qualified_NS1_family(self): """Test ':wikidata:testwiki:Talk:Q6' on enwp is namespace 1.""" @@ -432,14 +353,9 @@ config.family = 'wikipedia' link = Link(':wikidata:testwiki:Talk:Q6') link.parse() - if show_failures: - self.assertEqual(link.site, self.get_site('test.wp')) - self.assertEqual(link.title, 'Q6') - self.assertEqual(link.namespace, 1) - else: - self.assertEqual(link.site, self.get_site('enwp')) - self.assertEqual(link.title, 'Wikidata:testwiki:Talk:Q6') - self.assertEqual(link.namespace, 0) + self.assertEqual(link.site, self.get_site('test.wp')) + self.assertEqual(link.title, 'Q6') + self.assertEqual(link.namespace, 1) class TestFullyQualifiedOneSiteFamilyExplicitLinkParser(TestCase): @@ -643,50 +559,20 @@ config.mylang = 'en' config.family = 'wikisource' link = Link('en:wikipedia:Main Page') - if show_failures: - link.parse() - else: - self.assertRaisesRegex( - pywikibot.Error, - "Improperly formatted interwiki link 'en:wikipedia:Main Page'", - link.parse) - if show_failures: - self.assertEqual(link.site, self.get_site('enwp')) - else: - self.assertEqual(link.site, self.get_site('enws')) - if show_failures: - self.assertEqual(link.title, 'Main Page') - self.assertEqual(link.namespace, 0) - else: - try: - link.title - except pywikibot.Error as e: - self.assertEqual(str(e), "Improperly formatted interwiki link 'en:wikipedia:Main Page'") + link.parse() + self.assertEqual(link.site, self.get_site('enwp')) + self.assertEqual(link.title, 'Main Page') + self.assertEqual(link.namespace, 0) def test_fully_qualified_NS1_code(self): """Test 'en:wikipedia:Main Page' on enws is namespace 1.""" config.mylang = 'en' config.family = 'wikisource' link = Link('en:wikipedia:Talk:Main Page') - if show_failures: - link.parse() - else: - self.assertRaisesRegex( - pywikibot.Error, - "Improperly formatted interwiki link 'en:wikipedia:Talk:Main Page'", - link.parse) - if show_failures: - self.assertEqual(link.site, self.get_site('enwp')) - else: - self.assertEqual(link.site, self.get_site('enws')) - if show_failures: - self.assertEqual(link.title, 'Main Page') - self.assertEqual(link.namespace, 1) - else: - try: - link.title - except pywikibot.Error as e: - self.assertEqual(str(e), "Improperly formatted interwiki link 'en:wikipedia:Talk:Main Page'") + link.parse() + self.assertEqual(link.site, self.get_site('enwp')) + self.assertEqual(link.title, 'Main Page') + self.assertEqual(link.namespace, 1) def test_fully_qualified_NS0_family(self): """Test 'wikipedia:en:Main Page' on enws is namespace 0.""" @@ -729,16 +615,9 @@ config.family = 'wikidata' link = Link('en:wikipedia:Main Page') link.parse() - if show_failures: - self.assertEqual(link.site, self.get_site('enwp')) - else: - self.assertEqual(link.site, self.get_site('wikidata')) - if show_failures: - self.assertEqual(link.title, 'Main Page') - self.assertEqual(link.namespace, 4) - else: - self.assertEqual(link.title, 'En:wikipedia:Main Page') - self.assertEqual(link.namespace, 0) + self.assertEqual(link.site, self.get_site('enwp')) + self.assertEqual(link.title, 'Main Page') + self.assertEqual(link.namespace, 4) def test_fully_qualified_NS1_code(self): """Test 'en:wikipedia:Talk:Main Page' on wikidata is not namespace 1.""" @@ -746,52 +625,29 @@ config.family = 'wikidata' link = Link('en:wikipedia:Talk:Main Page') link.parse() - if show_failures: - self.assertEqual(link.site, self.get_site('enwp')) - else: - self.assertEqual(link.site, self.get_site('wikidata')) - if show_failures: - self.assertEqual(link.title, 'Talk:Main Page') - self.assertEqual(link.namespace, 4) - else: - self.assertEqual(link.title, 'En:wikipedia:Talk:Main Page') - self.assertEqual(link.namespace, 0) + self.assertEqual(link.site, self.get_site('enwp')) + self.assertEqual(link.title, 'Talk:Main Page') + self.assertEqual(link.namespace, 4) def test_fully_qualified_NS0_family(self): """Test 'wikipedia:en:Main Page' on wikidata is namespace 0.""" config.mylang = 'wikidata' config.family = 'wikidata' link = Link('wikipedia:en:Main Page') - if show_failures: - link.parse() - self.assertEqual(link.site, self.get_site('enwp')) - self.assertEqual(link.namespace, 0) - self.assertEqual(link.title, 'Main Page') - else: - self.assertRaisesRegex( - pywikibot.NoSuchSite, - 'Language wikidata does not exist in family wikipedia', - link.parse) # very bad + link.parse() + self.assertEqual(link.site, self.get_site('enwp')) + self.assertEqual(link.namespace, 0) + self.assertEqual(link.title, 'Main Page') def test_fully_qualified_NS1_family(self): """Test 'wikipedia:en:Talk:Main Page' on wikidata is namespace 1.""" config.mylang = 'wikidata' config.family = 'wikidata' link = Link('wikipedia:en:Talk:Main Page') - if show_failures: - link.parse() - else: - self.assertRaisesRegex( - pywikibot.NoSuchSite, - 'Language wikidata does not exist in family wikipedia', - link.parse) # very bad - if show_failures: - self.assertEqual(link.site, self.get_site('enwp')) - else: - self.assertEqual(link.site, self.get_site('wikidata')) - if show_failures: - self.assertEqual(link.title, 'Main Page') - self.assertEqual(link.namespace, 1) + link.parse() + self.assertEqual(link.site, self.get_site('enwp')) + self.assertEqual(link.title, 'Main Page') + self.assertEqual(link.namespace, 1) class TestFullyQualifiedNoLangFamilyImplicitLinkParser(TestCase): @@ -805,32 +661,20 @@ config.mylang = 'en' config.family = 'wikipedia' link = Link('testwiki:wikidata:Q6') - if show_failures: - link.parse() - self.assertEqual(link.site, pywikibot.Site('wikidata', 'wikidata')) - self.assertEqual(link.title, 'Q6') - self.assertEqual(link.namespace, 0) - else: - self.assertRaisesRegex( - pywikibot.Error, - 'Family testwiki does not exist', - link.parse) # very bad + link.parse() + self.assertEqual(link.site, pywikibot.Site('wikidata', 'wikidata')) + self.assertEqual(link.title, 'Q6') + self.assertEqual(link.namespace, 0) def test_fully_qualified_NS1_code(self): """Test 'testwiki:wikidata:Talk:Q6' on enwp is namespace 1.""" config.mylang = 'en' config.family = 'wikipedia' link = Link('testwiki:wikidata:Talk:Q6') - if show_failures: - link.parse() - self.assertEqual(link.site, pywikibot.Site('wikidata', 'wikidata')) - self.assertEqual(link.title, 'Q6') - self.assertEqual(link.namespace, 1) - else: - self.assertRaisesRegex( - pywikibot.Error, - 'Family testwiki does not exist', - link.parse) # very bad + link.parse() + self.assertEqual(link.site, pywikibot.Site('wikidata', 'wikidata')) + self.assertEqual(link.title, 'Q6') + self.assertEqual(link.namespace, 1) def test_fully_qualified_NS0_family(self): """Test 'wikidata:testwiki:Q6' on enwp is namespace 0.""" @@ -838,14 +682,9 @@ config.family = 'wikipedia' link = Link('wikidata:testwiki:Q6') link.parse() - if show_failures: - self.assertEqual(link.site, pywikibot.Site('test', 'wikipedia')) - self.assertEqual(link.title, 'Q6') - self.assertEqual(link.namespace, 0) - else: - self.assertEqual(link.site, pywikibot.Site('en', 'wikipedia')) - self.assertEqual(link.title, 'Wikidata:testwiki:Q6') - self.assertEqual(link.namespace, 0) + self.assertEqual(link.site, pywikibot.Site('test', 'wikipedia')) + self.assertEqual(link.title, 'Q6') + self.assertEqual(link.namespace, 0) def test_fully_qualified_NS1_family(self): """Test 'wikidata:testwiki:Talk:Q6' on enwp is namespace 1.""" @@ -853,14 +692,9 @@ config.family = 'wikipedia' link = Link('wikidata:testwiki:Talk:Q6') link.parse() - if show_failures: - self.assertEqual(link.site, pywikibot.Site('test', 'wikipedia')) - self.assertEqual(link.title, 'Q6') - self.assertEqual(link.namespace, 1) - else: - self.assertEqual(link.site, pywikibot.Site('en', 'wikipedia')) - self.assertEqual(link.title, 'Wikidata:testwiki:Talk:Q6') - self.assertEqual(link.namespace, 0) + self.assertEqual(link.site, pywikibot.Site('test', 'wikipedia')) + self.assertEqual(link.title, 'Q6') + self.assertEqual(link.namespace, 1) class TestFullyQualifiedOneSiteFamilyImplicitLinkParser(TestCase): @@ -910,6 +744,74 @@ self.assertEqual(link.namespace, 1) +class TestEmptyTitle(TestCase): + + """Test links which contain no title.""" + + family = 'wikipedia' + code = 'en' + + def test_interwiki_mainpage(self): + """Test that Link allow links without a title to the main page.""" + link = Link('en:', self.get_site()) + link.parse() + self.assertEqual(link.site, self.get_site()) + self.assertEqual(link.title, '') + self.assertEqual(link.namespace, 0) + + def test_interwiki_namespace_without_title(self): + """Test that Link doesn't allow links without a title.""" + link = Link('en:Help:', self.get_site()) + self.assertRaisesRegex( + InvalidTitle, "'en:Help:' has no title.", link.parse) + + def test_no_text(self): + """Test that Link doesn't allow empty.""" + link = Link('', self.get_site()) + self.assertRaisesRegex( + InvalidTitle, "The link does not contain a page title", link.parse) + + def test_namespace_lookalike(self): + """Test that Link does only detect valid namespaces.""" + link = Link('CAT:', self.get_site()) + link.parse() + self.assertEqual(link.site, self.get_site()) + self.assertEqual(link.title, 'CAT:') + self.assertEqual(link.namespace, 0) + + link = Link('en:CAT:', self.get_site()) + link.parse() + self.assertEqual(link.site, self.get_site()) + self.assertEqual(link.title, 'CAT:') + self.assertEqual(link.namespace, 0) + + +class TestInvalidInterwikiLinks(TestCase): + + """Test links to non-wikis.""" + + family = 'wikipedia' + code = 'en' + + def test_non_wiki_prefix(self): + """Test that Link fails if the interwiki prefix is not a wiki.""" + link = Link('bugzilla:1337') + self.assertRaisesRegex( + Error, + 'bugzilla:1337 is not a local page on wikipedia:en, and the ' + 'interwiki prefix bugzilla is not supported by PyWikiBot!', + link.parse) + + def test_other_wiki_prefix(self): + """Test that Link fails if the interwiki prefix is a unknown family.""" + link = Link('bulba:this-will-never-work') + self.assertRaisesRegex( + Error, + 'bulba:this-will-never-work is not a local page on wikipedia:en, ' + 'and the interwiki prefix bulba is not supported by PyWikiBot!', + link.parse) + + if __name__ == '__main__': try: unittest.main() -- To view, visit https://gerrit.wikimedia.org/r/161671 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Iec4f32fdefde15b2330dbb191fb3bbb0d04803e2 Gerrit-PatchSet: 12 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: Zhuyifei1999 <zhuyifei1999(a)gmail.com> Gerrit-Reviewer: jenkins-bot <>

1 0

[Gerrit] Add config.pickle_protocol, default to version 2 - change (pywikibot/core)
by jenkins-bot (Code Review) 12 Oct '14

12 Oct '14

jenkins-bot has submitted this change and it was merged. Change subject: Add config.pickle_protocol, default to version 2 ...................................................................... Add config.pickle_protocol, default to version 2 Pickle protocol version 2 has been implicitly used by Pywikibot since that is the default version for Python 2. Now that the package supports Python 3, which defaults to version 3 and has a version 4 also available, pickling needs to explicitly use protocol 2 in order that dumps may be accessed by either supported version of Python. Change-Id: Ib675990f9accd89b610c2bab1d05e90b29ae4baa --- M pywikibot/config2.py M pywikibot/data/api.py M pywikibot/data/wikidataquery.py M scripts/category.py M scripts/category_redirect.py M scripts/featured.py M scripts/interwiki.py M scripts/watchlist.py M scripts/weblinkchecker.py M scripts/welcome.py M tests/page_tests.py M tests/site_tests.py 12 files changed, 26 insertions(+), 18 deletions(-) Approvals: XZise: Looks good to me, but someone else must approve Mpaa: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/config2.py b/pywikibot/config2.py index 0b04b5a..19cc611 100644 --- a/pywikibot/config2.py +++ b/pywikibot/config2.py @@ -669,6 +669,13 @@ # if the user has already installed the library. use_mwparserfromhell = True +# Pickle protocol version to use for storing dumps. +# This config variable is not used for loading dumps. +# Version 2 is common to both Python 2 and 3, and should +# be used when dumps are accessed by both versions. +# Version 4 is only available for Python 3.4 +pickle_protocol = 2 + # End of configuration section # ============================ diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index 47cb74d..a43b0c0 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -752,7 +752,7 @@ """Write data to self._cachefile_path().""" data = [self._uniquedescriptionstr(), data, datetime.datetime.now()] with open(self._cachefile_path(), 'wb') as f: - pickle.dump(data, f) + pickle.dump(data, f, protocol=config.pickle_protocol) def submit(self): cached_available = self._load_cache() diff --git a/pywikibot/data/wikidataquery.py b/pywikibot/data/wikidataquery.py index 33f6f7d..f808053 100644 --- a/pywikibot/data/wikidataquery.py +++ b/pywikibot/data/wikidataquery.py @@ -20,6 +20,7 @@ from pywikibot.page import ItemPage, PropertyPage, Claim import pywikibot +from pywikibot import config def listify(x): @@ -519,7 +520,7 @@ with open(cacheFile, 'wb') as f: try: - pickle.dump(data, f) + pickle.dump(data, f, protocol=config.pickle_protocol) except IOError: pywikibot.warning(u"Failed to write cache file %s" % cacheFile) diff --git a/scripts/category.py b/scripts/category.py index 12beae3..15909f5 100755 --- a/scripts/category.py +++ b/scripts/category.py @@ -250,7 +250,7 @@ } # store dump to disk in binary format try: - pickle.dump(databases, f, protocol=pickle.HIGHEST_PROTOCOL) + pickle.dump(databases, f, protocol=config.pickle_protocol) except pickle.PicklingError: pass f.close() diff --git a/scripts/category_redirect.py b/scripts/category_redirect.py index b2123b2..b6e946f 100755 --- a/scripts/category_redirect.py +++ b/scripts/category_redirect.py @@ -26,8 +26,7 @@ import time from datetime import datetime, timedelta import pywikibot -from pywikibot import pagegenerators -from pywikibot import i18n +from pywikibot import i18n, pagegenerators, config if sys.version_info[0] > 2: import pickle as cPickle @@ -193,7 +192,7 @@ record = {} if record: with open(datafile + ".bak", "wb") as f: - cPickle.dump(record, f, -1) + cPickle.dump(record, f, protocol=config.pickle_protocol) try: template_list = self.site.family.category_redirect_templates[ self.site.code] @@ -394,7 +393,7 @@ pass with open(datafile, "wb") as f: - cPickle.dump(record, f, -1) + cPickle.dump(record, f, protocol=config.pickle_protocol) self.log_text.sort() problems.sort() diff --git a/scripts/featured.py b/scripts/featured.py index 98cb0a7..c2ff42b 100644 --- a/scripts/featured.py +++ b/scripts/featured.py @@ -66,7 +66,7 @@ import re import sys import pywikibot -from pywikibot import i18n, textlib +from pywikibot import i18n, textlib, config from pywikibot.pagegenerators import PreloadingGenerator from pywikibot.config2 import LS # line separator @@ -320,9 +320,8 @@ if not self.getOption('nocache') is True: pywikibot.output(u'Writing %d items to cache file %s.' % (len(self.cache), self.filename)) - f = open(self.filename, "wb") - pickle.dump(self.cache, f) - f.close() + with open(self.filename, "wb") as f: + pickle.dump(self.cache, f, protocol=config.pickle_protocol) self.cache = dict() def run(self): diff --git a/scripts/interwiki.py b/scripts/interwiki.py index ae17496..56679d1 100755 --- a/scripts/interwiki.py +++ b/scripts/interwiki.py @@ -2122,7 +2122,7 @@ mode = 'written' titles = [s.originPage.title() for s in self.subjects] with open(dumpfn, mode[0] + 'b') as f: - pickle.dump(titles, f) + pickle.dump(titles, f, protocol=config.pickle_protocol) pywikibot.output(u'Dump %s (%s) %s.' % (site.lang, site.family.name, mode)) return dumpfn diff --git a/scripts/watchlist.py b/scripts/watchlist.py index fcb2200..140b858 100755 --- a/scripts/watchlist.py +++ b/scripts/watchlist.py @@ -104,7 +104,7 @@ % (site.family.name, site.code, '-sysop' if sysop else '')), 'wb') as f: - pickle.dump(watchlist, f) + pickle.dump(watchlist, f, protocol=config.pickle_protocol) def refresh_all(new=False, sysop=False): diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py index 2a4d991..80d7415 100644 --- a/scripts/weblinkchecker.py +++ b/scripts/weblinkchecker.py @@ -621,8 +621,8 @@ def save(self): """ Save the .dat file to disk. """ - with open(self.datfilename, 'wb') as datfile: - pickle.dump(self.historyDict, datfile) + with open(self.datfilename, 'wb') as f: + pickle.dump(self.historyDict, f, protocol=config.pickle_protocol) class DeadLinkReportThread(threading.Thread): diff --git a/scripts/welcome.py b/scripts/welcome.py index c603eba..3ae71b0 100644 --- a/scripts/welcome.py +++ b/scripts/welcome.py @@ -997,7 +997,7 @@ else: import cPickle with open(filename, 'wb') as f: - cPickle.dump(bot.welcomed_users, f) + cPickle.dump(bot.welcomed_users, f, protocol=config.pickle_protocol) if __name__ == "__main__": main() diff --git a/tests/page_tests.py b/tests/page_tests.py index eed2993..c5213f5 100644 --- a/tests/page_tests.py +++ b/tests/page_tests.py @@ -9,6 +9,7 @@ import sys import pywikibot +from pywikibot import config import pywikibot.page from tests.aspects import unittest, TestCase, DefaultSiteTestCase @@ -438,7 +439,7 @@ def testPickleAbility(self): mainpage = self.get_mainpage() import pickle - mainpage_str = pickle.dumps(mainpage) + mainpage_str = pickle.dumps(mainpage, protocol=config.pickle_protocol) mainpage_unpickled = pickle.loads(mainpage_str) self.assertEqual(mainpage, mainpage_unpickled) diff --git a/tests/site_tests.py b/tests/site_tests.py index c9b99bf..07cbb27 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -14,6 +14,7 @@ import re import pywikibot +from pywikibot import config from pywikibot.tools import MediaWikiVersion as LV from pywikibot.data import api from tests.aspects import ( @@ -67,7 +68,7 @@ def testPickleAbility(self): import pickle mysite = self.get_site() - mysite_str = pickle.dumps(mysite) + mysite_str = pickle.dumps(mysite, protocol=config.pickle_protocol) mysite_pickled = pickle.loads(mysite_str) self.assertEqual(mysite, mysite_pickled) -- To view, visit https://gerrit.wikimedia.org/r/165422 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ib675990f9accd89b610c2bab1d05e90b29ae4baa Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de> Gerrit-Reviewer: jenkins-bot <>

1 0

← Newer
1
...
8
9
10
11
12
13
14
15
16
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

Pywikibot-commits October 2014