jenkins-bot has submitted this change and it was merged.

Change subject: [FEAT] Modify the code to be Python 3 compatible ......................................................................

[FEAT] Modify the code to be Python 3 compatible

- Use UnicodeMixin instead of a separate __str__ method - Replace __cmp__ with _cmpkey and ComparableMixin - Use bytes.decode instead of unicode(,) - Use BytesIO instead of StringIO if it's binary data - Open binary files in binary mode - Only decode/encode std* in Py2 - Always have no 'space' before \n in json strings (< Py3.4) - Map basestring to (str,) in scripts (in Py3) - Map xrange to range in casechecker (in Py3) - Use future print_function if necessary - Decode password file as UTF8

Change-Id: Ia7d416a578cc0fb52eaf1e2426244a693395738a --- M pywikibot/__init__.py M pywikibot/bot.py M pywikibot/botirc.py M pywikibot/data/wikidataquery.py M pywikibot/login.py M pywikibot/page.py M pywikibot/site.py M pywikibot/userinterfaces/terminal_interface_base.py M scripts/casechecker.py M scripts/category.py M scripts/checkimages.py M scripts/data_ingestion.py M scripts/editarticle.py M scripts/replace.py M scripts/weblinkchecker.py M tests/archivebot_tests.py M tests/data_ingestion_tests.py M tests/namespace_tests.py M tests/script_tests.py M tests/site_tests.py M tests/wikibase_tests.py 21 files changed, 126 insertions(+), 86 deletions(-)

Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified

diff --git a/pywikibot/__init__.py b/pywikibot/__init__.py index a5b964d..5257f65 100644 --- a/pywikibot/__init__.py +++ b/pywikibot/__init__.py @@ -378,7 +378,8 @@ ts[u'calendarmodel'])

def __str__(self): - return json.dumps(self.toWikibase(), indent=4, sort_keys=True) + return json.dumps(self.toWikibase(), indent=4, sort_keys=True, + separators=(',', ': '))

def __eq__(self, other): return self.__dict__ == other.__dict__ @@ -446,7 +447,8 @@ return WbQuantity(amount, wb['unit'], error)

def __str__(self): - return json.dumps(self.toWikibase(), indent=4, sort_keys=True) + return json.dumps(self.toWikibase(), indent=4, sort_keys=True, + separators=(',', ': '))

def __eq__(self, other): return self.__dict__ == other.__dict__ diff --git a/pywikibot/bot.py b/pywikibot/bot.py index 357bc40..2be8a2f 100644 --- a/pywikibot/bot.py +++ b/pywikibot/bot.py @@ -132,7 +132,7 @@ """ strExc = logging.Formatter.formatException(self, ei)

- if isinstance(strExc, str): + if sys.version_info[0] < 3 and isinstance(strExc, str): return strExc.decode(config.console_encoding) + '\n' else: return strExc + '\n' @@ -757,7 +757,9 @@ ''' % module_name try: module = __import__('%s' % module_name) - helpText = module.__doc__.decode('utf-8') + helpText = module.__doc__ + if sys.version_info[0] < 3: + helpText = helpText.decode('utf-8') if hasattr(module, 'docuReplacements'): for key, value in module.docuReplacements.items(): helpText = helpText.replace(key, value.strip('\n\r')) diff --git a/pywikibot/botirc.py b/pywikibot/botirc.py index 11a27a1..7ec9b99 100644 --- a/pywikibot/botirc.py +++ b/pywikibot/botirc.py @@ -74,7 +74,7 @@ if not ('N' in match.group('flags')): return try: - msg = unicode(e.arguments()[0], 'utf-8') + msg = e.arguments()[0].decode('utf-8') except UnicodeDecodeError: return if self.other_ns.match(msg): diff --git a/pywikibot/data/wikidataquery.py b/pywikibot/data/wikidataquery.py index 2794892..33f6f7d 100644 --- a/pywikibot/data/wikidataquery.py +++ b/pywikibot/data/wikidataquery.py @@ -464,7 +464,7 @@ """ Encode a query into a unique and universally safe format. """ - encQuery = hashlib.sha1(queryStr).hexdigest() + ".wdq_cache" + encQuery = hashlib.sha1(queryStr.encode('utf8')).hexdigest() + ".wdq_cache" return os.path.join(self.cacheDir, encQuery)

def readFromCache(self, queryStr): diff --git a/pywikibot/login.py b/pywikibot/login.py index feece17..2c864e6 100644 --- a/pywikibot/login.py +++ b/pywikibot/login.py @@ -9,6 +9,7 @@ # __version__ = '$Id$' # +import codecs

import pywikibot from pywikibot import config @@ -146,11 +147,11 @@ (u"wikipedia", u"my_wikipedia_user", u"my_wikipedia_pass") (u"en", u"wikipedia", u"my_en_wikipedia_user", u"my_en_wikipedia_pass") """ - password_f = open(config.password_file) + password_f = codecs.open(config.password_file, encoding='utf-8') for line in password_f: if not line.strip(): continue - entry = eval(line.decode('utf-8')) + entry = eval(line) if len(entry) == 4: # for userinfo included code and family if entry[0] == self.site.code and \ entry[1] == self.site.family.name and \ diff --git a/pywikibot/page.py b/pywikibot/page.py index 1248b72..39cb3ad 100644 --- a/pywikibot/page.py +++ b/pywikibot/page.py @@ -32,6 +32,7 @@ from urllib import urlopen else: unicode = basestring = str + long = int from html import entities as htmlentitydefs from urllib.parse import quote_from_bytes, unquote_to_bytes from urllib.request import urlopen @@ -2863,7 +2864,7 @@ if diffto and 'aliases' in diffto: for lang in set(diffto['aliases'].keys()) - set(aliases.keys()): aliases[lang] = [] - for lang, strings in aliases.items(): + for lang, strings in list(aliases.items()): if diffto and 'aliases' in diffto and lang in diffto['aliases']: empty = len(diffto['aliases'][lang]) - len(strings) if empty > 0: @@ -3622,7 +3623,7 @@ if len(self.qualifiers) > 0: data['qualifiers'] = {} data['qualifiers-order'] = list(self.qualifiers.keys()) - for prop, qualifiers in self.qualifiers.iteritems(): + for prop, qualifiers in self.qualifiers.items(): for qualifier in qualifiers: qualifier.isQualifier = True data['qualifiers'][prop] = [qualifier.toJSON() for qualifier in qualifiers] @@ -3630,7 +3631,7 @@ data['references'] = [] for collection in self.sources: reference = {'snaks': {}, 'snaks-order': list(collection.keys())} - for prop, val in collection.iteritems(): + for prop, val in collection.items(): reference['snaks'][prop] = [] for source in val: source.isReference = True @@ -4237,9 +4238,14 @@ self.site.code, title)

- def __str__(self): - """Return a string representation.""" - return self.astext().encode("ascii", "backslashreplace") + if sys.version_info[0] > 2: + def __str__(self): + """Return a string representation.""" + return self.__unicode__() + else: + def __str__(self): + """Return a string representation.""" + return self.astext().encode("ascii", "backslashreplace")

def _cmpkey(self): """ diff --git a/pywikibot/site.py b/pywikibot/site.py index 20c1f03..4967aa0 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -29,7 +29,9 @@ import pywikibot from pywikibot import config from pywikibot.family import AutoFamily -from pywikibot.tools import itergroup, deprecated, deprecate_arg +from pywikibot.tools import ( + itergroup, deprecated, deprecate_arg, UnicodeMixin, ComparableMixin, +) from pywikibot.throttle import Throttle from pywikibot.data import api from pywikibot.exceptions import ( @@ -146,7 +148,7 @@ return _families[fam]

-class Namespace(Iterable): +class Namespace(Iterable, ComparableMixin, UnicodeMixin):

""" Namespace site data object.

@@ -306,26 +308,31 @@ else: return self.aliases[index - 1]

- def __str__(self): - """Return a string representation.""" - if sys.version_info[0] > 2: - return self.__unicode__() - - if self.id == 0: - return ':' - elif self.id in (6, 14): - return ':' + self.canonical_name + ':' + @staticmethod + def _colons(id, name): + """Return the name with required colons, depending on the ID.""" + if id == 0: + return u':' + elif id in (6, 14): + return u':' + name + u':' else: - return self.canonical_name + ':' + return u'' + name + u':' + + def __str__(self): + """Return a the canonical string representation.""" + return self.canonical_prefix()

def __unicode__(self): - """Return a unicode string representation.""" - if self.id == 0: - return u':' - elif self.id in (6, 14): - return u':' + self.custom_name + u':' - else: - return u'' + self.custom_name + u':' + """Return a the custom string representation.""" + return self.custom_prefix() + + def canonical_prefix(self): + """Return the canonical name with required colons.""" + return Namespace._colons(self.id, self.canonical_name) + + def custom_prefix(self): + """Return the custom name with required colons.""" + return Namespace._colons(self.id, self.custom_name)

def __index__(self): return self.id @@ -348,14 +355,9 @@ else: return True

- def __cmp__(self, other): - """Compare two namespace ids.""" - if self.id == other.id: - return 0 - elif self.id > other.id: - return 1 - else: - return -1 + def _cmpkey(self): + """Return the ID as a comparison key.""" + return self.id

def __repr__(self): """Return a reconstructable representation.""" @@ -425,7 +427,7 @@ return None

-class BaseSite(object): +class BaseSite(ComparableMixin):

"""Site methods that are independent of the communication interface."""

@@ -519,13 +521,9 @@ """ return self.__code

- def __cmp__(self, other): + def _cmpkey(self): """Perform equality and inequality tests on Site objects.""" - if not isinstance(other, BaseSite): - return 1 - if self.family == other.family: - return cmp(self.code, other.code) - return cmp(self.family.name, other.family.name) + return (self.family.name, self.code)

def __getstate__(self): """ Remove Lock based classes before pickling. """ diff --git a/pywikibot/userinterfaces/terminal_interface_base.py b/pywikibot/userinterfaces/terminal_interface_base.py index 03b51f8..89214a5 100755 --- a/pywikibot/userinterfaces/terminal_interface_base.py +++ b/pywikibot/userinterfaces/terminal_interface_base.py @@ -102,7 +102,9 @@ line, count = colorTagR.subn('', line) if count > 0: line += ' ***' - targetStream.write(line.encode(self.encoding, 'replace')) + if sys.version_info[0] < 3: + line = line.encode(self.encoding, 'replace') + targetStream.write(line)

printColorized = printNonColorized

@@ -198,7 +200,8 @@ text = self._raw_input() except KeyboardInterrupt: raise pywikibot.QuitKeyboardInterrupt() - text = unicode(text, self.encoding) + if sys.version_info[0] < 3: + text = text.decode(self.encoding) return text

def inputChoice(self, question, options, hotkeys, default=None): diff --git a/scripts/casechecker.py b/scripts/casechecker.py index 75e55c6..7d0c4da 100644 --- a/scripts/casechecker.py +++ b/scripts/casechecker.py @@ -8,6 +8,7 @@ # # Distributed under the terms of the MIT license. # +from __future__ import print_function __version__ = '$Id$'

import os @@ -17,6 +18,9 @@ import pywikibot from pywikibot import i18n from pywikibot.data import api + +if sys.version_info[0] > 2: + xrange = range

# @@ -69,11 +73,11 @@ pass

if color == FOREGROUND_BLUE: - print('(b:'), + print('(b:', end=' ') if color == FOREGROUND_GREEN: - print('(g:'), + print('(g:', end=' ') if color == FOREGROUND_RED: - print('(r:'), + print('(r:', end=' ')

# end of console code

diff --git a/scripts/category.py b/scripts/category.py index 246d637..d5044a8 100755 --- a/scripts/category.py +++ b/scripts/category.py @@ -103,11 +103,16 @@ import re import pickle import bz2 +import sys + import pywikibot from pywikibot import config, pagegenerators from pywikibot import i18n, textlib from pywikibot.tools import deprecate_arg, deprecated

+if sys.version_info[0] > 2: + basestring = (str, ) + # This is required for the text that is shown when you run this script # with the parameter -help. docuReplacements = { diff --git a/scripts/checkimages.py b/scripts/checkimages.py index 8004226..c851382 100644 --- a/scripts/checkimages.py +++ b/scripts/checkimages.py @@ -95,10 +95,15 @@ import time import datetime import locale +import sys + import pywikibot from pywikibot import pagegenerators as pg from pywikibot import config, i18n

+if sys.version_info[0] > 2: + basestring = (str, ) + locale.setlocale(locale.LC_ALL, '')

############################################################################### diff --git a/scripts/data_ingestion.py b/scripts/data_ingestion.py index e62ce32..70856eb 100755 --- a/scripts/data_ingestion.py +++ b/scripts/data_ingestion.py @@ -13,6 +13,7 @@ import hashlib import base64 import sys +import io

import pywikibot # TODO: nosetests3 fails on 'import <other_script>', which is used by many @@ -23,11 +24,9 @@ if sys.version_info[0] > 2: from urllib.parse import urlparse from urllib.request import urlopen - import io as StringIO else: from urlparse import urlparse from urllib import urlopen - import StringIO

class Photo(object): @@ -55,13 +54,13 @@

def downloadPhoto(self): """ - Download the photo and store it in a StringIO.StringIO object. + Download the photo and store it in a io.BytesIO object.

TODO: Add exception handling """ if not self.contents: imageFile = urlopen(self.URL).read() - self.contents = StringIO.StringIO(imageFile) + self.contents = io.BytesIO(imageFile) return self.contents

def findDuplicateImages(self, @@ -189,12 +188,12 @@

def downloadPhoto(self, photoUrl=''): """ - Download the photo and store it in a StrinIO.StringIO object. + Download the photo and store it in a io.BytesIO object.

TODO: Add exception handling """ imageFile = urlopen(photoUrl).read() - return StringIO.StringIO(imageFile) + return io.BytesIO(imageFile)

def findDuplicateImages(self, photo=None, site=pywikibot.Site(u'commons', u'commons')): """ diff --git a/scripts/editarticle.py b/scripts/editarticle.py index 15b6eb2..74af631 100755 --- a/scripts/editarticle.py +++ b/scripts/editarticle.py @@ -19,7 +19,6 @@ #

import os -import string import optparse import tempfile

@@ -30,7 +29,8 @@

class ArticleEditor(object): # join lines if line starts with this ones - joinchars = string.letters + '[]' + string.digits + # TODO: No apparent usage + # joinchars = string.letters + '[]' + string.digits

def __init__(self, *args): self.set_options(*args) diff --git a/scripts/replace.py b/scripts/replace.py index 9d92a1e..5c9e750 100755 --- a/scripts/replace.py +++ b/scripts/replace.py @@ -125,14 +125,19 @@

import re import time +import webbrowser +import sys + import pywikibot from pywikibot import i18n, textlib, pagegenerators, Bot from pywikibot import editor as editarticle -import webbrowser

# Imports predefined replacements tasks from fixes.py from pywikibot import fixes

+if sys.version_info[0] > 2: + basestring = (str, ) + # This is required for the text that is shown when you run this script # with the parameter -help. docuReplacements = { diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py index 5d1158e..2a4d991 100644 --- a/scripts/weblinkchecker.py +++ b/scripts/weblinkchecker.py @@ -112,6 +112,7 @@ import urllib.parse as urlparse import urllib.request as urllib import http.client as httplib + basestring = (str, ) else: import urlparse import urllib diff --git a/tests/archivebot_tests.py b/tests/archivebot_tests.py index ba1a8ca..abe42aa 100644 --- a/tests/archivebot_tests.py +++ b/tests/archivebot_tests.py @@ -8,12 +8,16 @@ __version__ = '$Id$'

from datetime import datetime +import sys import pywikibot import pywikibot.page from pywikibot.textlib import TimeStripper from scripts import archivebot from tests.aspects import unittest, TestCase

+if sys.version_info[0] > 2: + basestring = (str,) + THREADS = { 'als': 4, 'ar': 1, 'bar': 0, 'bg': 0, 'bjn': 1, 'bs': 0, 'ca': 5, 'ckb': 2, 'cs': 0, 'de': 7, 'en': 25, 'eo': 1, 'es': 13, 'fa': 2, 'fr': 25, 'frr': 2, diff --git a/tests/data_ingestion_tests.py b/tests/data_ingestion_tests.py index 741ccef..f7f6711 100644 --- a/tests/data_ingestion_tests.py +++ b/tests/data_ingestion_tests.py @@ -25,8 +25,8 @@ )

def test_downloadPhoto(self): - f = open(os.path.join(os.path.split(__file__)[0], 'data', 'MP_sounds.png')) - self.assertEqual(f.read(), self.obj.downloadPhoto().read()) + with open(os.path.join(os.path.split(__file__)[0], 'data', 'MP_sounds.png'), 'rb') as f: + self.assertEqual(f.read(), self.obj.downloadPhoto().read())

def test_findDuplicateImages(self): duplicates = self.obj.findDuplicateImages() diff --git a/tests/namespace_tests.py b/tests/namespace_tests.py index 73930e3..5dbd47d 100644 --- a/tests/namespace_tests.py +++ b/tests/namespace_tests.py @@ -129,11 +129,11 @@ y = Namespace(id=6, custom_name=u'ملف', canonical_name=u'File', aliases=[u'Image', u'Immagine'], **kwargs)

- if sys.version_info[0] == 2: - self.assertEqual(str(y), ':File:') + self.assertEqual(str(y), ':File:') + if sys.version_info[0] <= 2: self.assertEqual(unicode(y), u':ملف:') - else: - self.assertEqual(str(y), u':ملف:') + self.assertEqual(y.canonical_prefix(), ':File:') + self.assertEqual(y.custom_prefix(), u':ملف:')

def testNamespaceCompare(self): a = Namespace(id=0, canonical_name=u'') @@ -166,10 +166,6 @@ self.assertEqual(x, u'Image')

self.assertEqual(y, u'ملف') - - # FIXME: Namespace is missing operators required for py3 - if sys.version_info[0] > 2: - return

self.assertLess(a, x) self.assertGreater(x, a) diff --git a/tests/script_tests.py b/tests/script_tests.py index e2324c5..7b26ed2 100644 --- a/tests/script_tests.py +++ b/tests/script_tests.py @@ -127,7 +127,6 @@ 'login': 'Logged in on ', 'pagefromfile': 'Please enter the file name', 'replace': 'Press Enter to use this default message', - 'replicate_wiki': 'error: too few arguments', 'script_wui': 'Pre-loading all relevant page contents', 'shell': 'Welcome to the', 'spamremove': 'No spam site specified', @@ -141,6 +140,11 @@ 'revertbot': 'Fetching new batch of contributions', 'upload': 'ERROR: Upload error', } + +if sys.version_info[0] > 2: + no_args_expected_results['replicate_wiki'] = 'error: the following arguments are required: destination' +else: + no_args_expected_results['replicate_wiki'] = 'error: too few arguments'

def collector(loader=unittest.loader.defaultTestLoader): @@ -184,6 +188,11 @@

def execute(command, data_in=None, timeout=0): """Execute a command and capture outputs.""" + def decode(stream): + if sys.version_info[0] > 2: + return stream.decode(config.console_encoding) + else: + return stream options = { 'stdout': subprocess.PIPE, 'stderr': subprocess.PIPE @@ -193,7 +202,10 @@

p = subprocess.Popen(command, **options) if data_in is not None: + if sys.version_info[0] > 2: + data_in = data_in.encode(config.console_encoding) p.stdin.write(data_in) + p.stdin.flush() # _communicate() otherwise has a broken pipe waited = 0 while waited < timeout and p.poll() is None: time.sleep(1) @@ -202,8 +214,8 @@ p.kill() data_out = p.communicate() return {'exit_code': p.returncode, - 'stdout': data_out[0], - 'stderr': data_out[1]} + 'stdout': decode(data_out[0]), + 'stderr': decode(data_out[1])}

class TestScriptMeta(MetaTestCaseClass): diff --git a/tests/site_tests.py b/tests/site_tests.py index b2fb360..3954141 100644 --- a/tests/site_tests.py +++ b/tests/site_tests.py @@ -951,10 +951,6 @@ self.assertLessEqual(len(dr['revisions']), 10) self.assertTrue(all(isinstance(rev, dict) for rev in dr['revisions'])) - dr2 = list(mysite.deletedrevs(page=mainpage, total=10))[0] - self.assertLessEqual(len(dr2['revisions']), 10) - self.assertTrue(all(isinstance(rev, dict) - for rev in dr2['revisions'])) for item in mysite.deletedrevs(start="2008-10-11T01:02:03Z", page=mainpage, total=5): for rev in item['revisions']: @@ -1303,7 +1299,8 @@ self.assertEqual(image_namespace.custom_name, 'Fil') self.assertEqual(image_namespace.canonical_name, 'File') self.assertEqual(str(image_namespace), ':File:') - self.assertEqual(unicode(image_namespace), ':Fil:') + self.assertEqual(image_namespace.custom_prefix(), ':Fil:') + self.assertEqual(image_namespace.canonical_prefix(), ':File:') self.assertEqual(image_namespace.aliases, ['Image']) self.assertEqual(len(image_namespace), 3)

diff --git a/tests/wikibase_tests.py b/tests/wikibase_tests.py index a4d7b6b..b288484 100644 --- a/tests/wikibase_tests.py +++ b/tests/wikibase_tests.py @@ -79,15 +79,15 @@ {'amount': 5, 'lowerBound': 2, 'upperBound': 7, 'unit': '1', }) q = pywikibot.WbQuantity(amount=0.044405586) - self.assertEqual(q.toWikibase(), - {'amount': 0.044405586, 'lowerBound': 0.044405586, - 'upperBound': 0.044405586, 'unit': '1', }) + q_dict = {'amount': 0.044405586, 'lowerBound': 0.044405586, + 'upperBound': 0.044405586, 'unit': '1', } + self.assertEqual(q.toWikibase(), q_dict) # test other WbQuantity methods self.assertEqual("%s" % q, '{\n' - ' "amount": %(val)r, \n' - ' "lowerBound": %(val)r, \n' - ' "unit": "1", \n' + ' "amount": %(val)r,\n' + ' "lowerBound": %(val)r,\n' + ' "unit": "1",\n' ' "upperBound": %(val)r\n' '}' % {'val': 0.044405586}) self.assertEqual("%r" % q,