jenkins-bot has submitted this change and it was merged.
Change subject: Add config.pickle_protocol, default to version 2
......................................................................
Add config.pickle_protocol, default to version 2
Pickle protocol version 2 has been implicitly used by Pywikibot
since that is the default version for Python 2. Now that the
package supports Python 3, which defaults to version 3 and has
a version 4 also available, pickling needs to explicitly use
protocol 2 in order that dumps may be accessed by either supported
version of Python.
Change-Id: Ib675990f9accd89b610c2bab1d05e90b29ae4baa
---
M pywikibot/config2.py
M pywikibot/data/api.py
M pywikibot/data/wikidataquery.py
M scripts/category.py
M scripts/category_redirect.py
M scripts/featured.py
M scripts/interwiki.py
M scripts/watchlist.py
M scripts/weblinkchecker.py
M scripts/welcome.py
M tests/page_tests.py
M tests/site_tests.py
12 files changed, 26 insertions(+), 18 deletions(-)
Approvals:
XZise: Looks good to me, but someone else must approve
Mpaa: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/config2.py b/pywikibot/config2.py
index 0b04b5a..19cc611 100644
--- a/pywikibot/config2.py
+++ b/pywikibot/config2.py
@@ -669,6 +669,13 @@
# if the user has already installed the library.
use_mwparserfromhell = True
+# Pickle protocol version to use for storing dumps.
+# This config variable is not used for loading dumps.
+# Version 2 is common to both Python 2 and 3, and should
+# be used when dumps are accessed by both versions.
+# Version 4 is only available for Python 3.4
+pickle_protocol = 2
+
# End of configuration section
# ============================
diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py
index 47cb74d..a43b0c0 100644
--- a/pywikibot/data/api.py
+++ b/pywikibot/data/api.py
@@ -752,7 +752,7 @@
"""Write data to self._cachefile_path()."""
data = [self._uniquedescriptionstr(), data, datetime.datetime.now()]
with open(self._cachefile_path(), 'wb') as f:
- pickle.dump(data, f)
+ pickle.dump(data, f, protocol=config.pickle_protocol)
def submit(self):
cached_available = self._load_cache()
diff --git a/pywikibot/data/wikidataquery.py b/pywikibot/data/wikidataquery.py
index 33f6f7d..f808053 100644
--- a/pywikibot/data/wikidataquery.py
+++ b/pywikibot/data/wikidataquery.py
@@ -20,6 +20,7 @@
from pywikibot.page import ItemPage, PropertyPage, Claim
import pywikibot
+from pywikibot import config
def listify(x):
@@ -519,7 +520,7 @@
with open(cacheFile, 'wb') as f:
try:
- pickle.dump(data, f)
+ pickle.dump(data, f, protocol=config.pickle_protocol)
except IOError:
pywikibot.warning(u"Failed to write cache file %s" %
cacheFile)
diff --git a/scripts/category.py b/scripts/category.py
index 12beae3..15909f5 100755
--- a/scripts/category.py
+++ b/scripts/category.py
@@ -250,7 +250,7 @@
}
# store dump to disk in binary format
try:
- pickle.dump(databases, f, protocol=pickle.HIGHEST_PROTOCOL)
+ pickle.dump(databases, f, protocol=config.pickle_protocol)
except pickle.PicklingError:
pass
f.close()
diff --git a/scripts/category_redirect.py b/scripts/category_redirect.py
index b2123b2..b6e946f 100755
--- a/scripts/category_redirect.py
+++ b/scripts/category_redirect.py
@@ -26,8 +26,7 @@
import time
from datetime import datetime, timedelta
import pywikibot
-from pywikibot import pagegenerators
-from pywikibot import i18n
+from pywikibot import i18n, pagegenerators, config
if sys.version_info[0] > 2:
import pickle as cPickle
@@ -193,7 +192,7 @@
record = {}
if record:
with open(datafile + ".bak", "wb") as f:
- cPickle.dump(record, f, -1)
+ cPickle.dump(record, f, protocol=config.pickle_protocol)
try:
template_list = self.site.family.category_redirect_templates[
self.site.code]
@@ -394,7 +393,7 @@
pass
with open(datafile, "wb") as f:
- cPickle.dump(record, f, -1)
+ cPickle.dump(record, f, protocol=config.pickle_protocol)
self.log_text.sort()
problems.sort()
diff --git a/scripts/featured.py b/scripts/featured.py
index 98cb0a7..c2ff42b 100644
--- a/scripts/featured.py
+++ b/scripts/featured.py
@@ -66,7 +66,7 @@
import re
import sys
import pywikibot
-from pywikibot import i18n, textlib
+from pywikibot import i18n, textlib, config
from pywikibot.pagegenerators import PreloadingGenerator
from pywikibot.config2 import LS # line separator
@@ -320,9 +320,8 @@
if not self.getOption('nocache') is True:
pywikibot.output(u'Writing %d items to cache file %s.'
% (len(self.cache), self.filename))
- f = open(self.filename, "wb")
- pickle.dump(self.cache, f)
- f.close()
+ with open(self.filename, "wb") as f:
+ pickle.dump(self.cache, f, protocol=config.pickle_protocol)
self.cache = dict()
def run(self):
diff --git a/scripts/interwiki.py b/scripts/interwiki.py
index ae17496..56679d1 100755
--- a/scripts/interwiki.py
+++ b/scripts/interwiki.py
@@ -2122,7 +2122,7 @@
mode = 'written'
titles = [s.originPage.title() for s in self.subjects]
with open(dumpfn, mode[0] + 'b') as f:
- pickle.dump(titles, f)
+ pickle.dump(titles, f, protocol=config.pickle_protocol)
pywikibot.output(u'Dump %s (%s) %s.' % (site.lang, site.family.name,
mode))
return dumpfn
diff --git a/scripts/watchlist.py b/scripts/watchlist.py
index fcb2200..140b858 100755
--- a/scripts/watchlist.py
+++ b/scripts/watchlist.py
@@ -104,7 +104,7 @@
% (site.family.name, site.code,
'-sysop' if sysop else '')),
'wb') as f:
- pickle.dump(watchlist, f)
+ pickle.dump(watchlist, f, protocol=config.pickle_protocol)
def refresh_all(new=False, sysop=False):
diff --git a/scripts/weblinkchecker.py b/scripts/weblinkchecker.py
index 2a4d991..80d7415 100644
--- a/scripts/weblinkchecker.py
+++ b/scripts/weblinkchecker.py
@@ -621,8 +621,8 @@
def save(self):
""" Save the .dat file to disk. """
- with open(self.datfilename, 'wb') as datfile:
- pickle.dump(self.historyDict, datfile)
+ with open(self.datfilename, 'wb') as f:
+ pickle.dump(self.historyDict, f, protocol=config.pickle_protocol)
class DeadLinkReportThread(threading.Thread):
diff --git a/scripts/welcome.py b/scripts/welcome.py
index c603eba..3ae71b0 100644
--- a/scripts/welcome.py
+++ b/scripts/welcome.py
@@ -997,7 +997,7 @@
else:
import cPickle
with open(filename, 'wb') as f:
- cPickle.dump(bot.welcomed_users, f)
+ cPickle.dump(bot.welcomed_users, f, protocol=config.pickle_protocol)
if __name__ == "__main__":
main()
diff --git a/tests/page_tests.py b/tests/page_tests.py
index eed2993..c5213f5 100644
--- a/tests/page_tests.py
+++ b/tests/page_tests.py
@@ -9,6 +9,7 @@
import sys
import pywikibot
+from pywikibot import config
import pywikibot.page
from tests.aspects import unittest, TestCase, DefaultSiteTestCase
@@ -438,7 +439,7 @@
def testPickleAbility(self):
mainpage = self.get_mainpage()
import pickle
- mainpage_str = pickle.dumps(mainpage)
+ mainpage_str = pickle.dumps(mainpage, protocol=config.pickle_protocol)
mainpage_unpickled = pickle.loads(mainpage_str)
self.assertEqual(mainpage, mainpage_unpickled)
diff --git a/tests/site_tests.py b/tests/site_tests.py
index c9b99bf..07cbb27 100644
--- a/tests/site_tests.py
+++ b/tests/site_tests.py
@@ -14,6 +14,7 @@
import re
import pywikibot
+from pywikibot import config
from pywikibot.tools import MediaWikiVersion as LV
from pywikibot.data import api
from tests.aspects import (
@@ -67,7 +68,7 @@
def testPickleAbility(self):
import pickle
mysite = self.get_site()
- mysite_str = pickle.dumps(mysite)
+ mysite_str = pickle.dumps(mysite, protocol=config.pickle_protocol)
mysite_pickled = pickle.loads(mysite_str)
self.assertEqual(mysite, mysite_pickled)
--
To view, visit
https://gerrit.wikimedia.org/r/165422
To unsubscribe, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ib675990f9accd89b610c2bab1d05e90b29ae4baa
Gerrit-PatchSet: 2
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>