jenkins-bot has submitted this change and it was merged.
Change subject: docs: clean up a set of errors
......................................................................
docs: clean up a set of errors
There are still a few issues remaining, most notably related to
the deprecation wrappers (a deprecated class becomes a function,
which confuses sphinx) and the magic that transforms script
docstrings.
Bug: T112491
Change-Id: I3c5b6bff67786c08797a8fe6df34a43e62eb952d
---
M docs/api_ref/pywikibot.families.rst
M docs/index.rst
M pywikibot/backports.py
M pywikibot/bot.py
M pywikibot/data/wikidataquery.py
M pywikibot/flow.py
M pywikibot/i18n.py
M pywikibot/login.py
M pywikibot/page.py
M pywikibot/site.py
10 files changed, 64 insertions(+), 50 deletions(-)
Approvals:
XZise: Looks good to me, approved
jenkins-bot: Verified
diff --git a/docs/api_ref/pywikibot.families.rst b/docs/api_ref/pywikibot.families.rst
index 58cacf4..fb96513 100644
--- a/docs/api_ref/pywikibot.families.rst
+++ b/docs/api_ref/pywikibot.families.rst
@@ -153,10 +153,10 @@
:undoc-members:
:show-inheritance:
-pywikibot.families.wikimedia_family module
-------------------------------------------
+pywikibot.families.wikimediachapter_family module
+-------------------------------------------------
-.. automodule:: pywikibot.families.wikimedia_family
+.. automodule:: pywikibot.families.wikimediachapter_family
:members:
:undoc-members:
:show-inheritance:
diff --git a/docs/index.rst b/docs/index.rst
index 3c1b87d..81e7ad6 100644
--- a/docs/index.rst
+++ b/docs/index.rst
@@ -46,6 +46,16 @@
getting_help
api_ref/index
+For framework developers:
+=========================
+
+.. toctree::
+ :maxdepth: 2
+
+ api_ref/tests/index
+ scripts.maintenance
+
+
Miscellaneous
=============
.. toctree::
diff --git a/pywikibot/backports.py b/pywikibot/backports.py
index fbebf8f..cb44b6e 100644
--- a/pywikibot/backports.py
+++ b/pywikibot/backports.py
@@ -3,8 +3,18 @@
This module contains backports to support older Python versions.
They contain the backported code originally developed for Python. It is
-therefore distributed under the PSF license, as follows:
+therefore distributed under the PSF license.
+"""
+#
+# (C) Python Software Foundation, 2001-2014
+# (C) with modifications from Pywikibot team, 2015
+#
+# Distributed under the terms of the PSF license.
+#
+from __future__ import unicode_literals
+
+__license__ = """
PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2
--------------------------------------------
1. This LICENSE AGREEMENT is between the Python Software Foundation
@@ -52,14 +62,6 @@
agrees to be bound by the terms and conditions of this License
Agreement.
"""
-#
-# (C) Python Software Foundation, 2001-2014
-# (C) with modifications from Pywikibot team, 2015
-#
-# Distributed under the terms of the PSF license.
-#
-from __future__ import unicode_literals
-
import logging
import warnings
diff --git a/pywikibot/bot.py b/pywikibot/bot.py
index dae9760..99a0d0f 100644
--- a/pywikibot/bot.py
+++ b/pywikibot/bot.py
@@ -2091,16 +2091,16 @@
@param data: data to be saved, or None if the diff should be created
automatically
@kwarg summary: revision comment, passed to ItemPage.editEntity
- @kwtype summary: str
+ @type summary: str
@kwarg show_diff: show changes between oldtext and newtext (default:
True)
- @kwtype show_diff: bool
+ @type show_diff: bool
@kwarg ignore_server_errors: if True, server errors will be reported
and ignored (default: False)
- @kwtype ignore_server_errors: bool
+ @type ignore_server_errors: bool
@kwarg ignore_save_related_errors: if True, errors related to
- page save will be reported and ignored (default: False)
- @kwtype ignore_save_related_errors: bool
+ page save will be reported and ignored (default: False)
+ @type ignore_save_related_errors: bool
"""
self.current_page = item
diff --git a/pywikibot/data/wikidataquery.py b/pywikibot/data/wikidataquery.py
index 59cb48a..054eb44 100644
--- a/pywikibot/data/wikidataquery.py
+++ b/pywikibot/data/wikidataquery.py
@@ -232,7 +232,7 @@
Sub-classes must override this method.
- @raise NotImplementedError: Always raised by this abstract method
+ @raises NotImplementedError: Always raised by this abstract method
"""
raise NotImplementedError
diff --git a/pywikibot/flow.py b/pywikibot/flow.py
index 27c59ce..46dfac9 100644
--- a/pywikibot/flow.py
+++ b/pywikibot/flow.py
@@ -46,8 +46,8 @@
@param title: normalized title of the page
@type title: unicode
- @raise TypeError: incorrect use of parameters
- @raise ValueError: use of non-Flow-enabled Site
+ @raises TypeError: incorrect use of parameters
+ @raises ValueError: use of non-Flow-enabled Site
"""
super(FlowPage, self).__init__(source, title)
@@ -194,8 +194,8 @@
@type topiclist_data: dict
@return: A Topic object derived from the supplied data
@rtype: Topic
- @raise TypeError: any passed parameters have wrong types
- @raise ValueError: the passed topiclist_data is missing required data
+ @raises TypeError: any passed parameters have wrong types
+ @raises ValueError: the passed topiclist_data is missing required data
"""
if not isinstance(board, Board):
raise TypeError('board must be a pywikibot.flow.Board object.')
@@ -276,7 +276,7 @@
@param uuid: UUID of a Flow post
@type uuid: unicode
- @raise TypeError: incorrect types of parameters
+ @raises TypeError: incorrect types of parameters
"""
if not isinstance(page, Topic):
raise TypeError('Page must be a Topic object')
@@ -303,8 +303,8 @@
@type data: dict
@return: A Post object
- @raise TypeError: data is not a dict
- @raise ValueError: data is missing required entries
+ @raises TypeError: data is not a dict
+ @raises ValueError: data is missing required entries
"""
post = cls(page, post_uuid)
post._set_data(data)
@@ -316,8 +316,8 @@
@param data: The data to store internally
@type data: dict
- @raise TypeError: data is not a dict
- @raise ValueError: missing data entries or post/revision not found
+ @raises TypeError: data is not a dict
+ @raises ValueError: missing data entries or post/revision not found
"""
if not isinstance(data, dict):
raise TypeError('Illegal post data (must be a dictionary).')
@@ -386,7 +386,7 @@
@type format: unicode
@return: The contents of the post in the given content format
@rtype: unicode
- @raise NotImplementedError: use of 'sysop'
+ @raises NotImplementedError: use of 'sysop'
"""
if sysop:
raise NotImplementedError
@@ -402,7 +402,7 @@
@type format: str ('wikitext', 'html', or 'fixed-html')
@param force: Whether to reload from the API instead of using the cache
@type force: bool
- @return This post's replies
+ @return: This post's replies
@rtype: list of Posts
"""
if format not in ('wikitext', 'html', 'fixed-html'):
diff --git a/pywikibot/i18n.py b/pywikibot/i18n.py
index 661c015..c8c516c 100644
--- a/pywikibot/i18n.py
+++ b/pywikibot/i18n.py
@@ -529,13 +529,13 @@
As an examples, if we had several json dictionaries in test folder like:
- en.json:
+ en.json::
{
"test-plural": "Bot: Changing %(num)s {{PLURAL:%(num)d|page|pages}}.",
}
- fr.json:
+ fr.json::
{
"test-plural": "Robot: Changer %(descr)s {{PLURAL:num|une page|quelques pages}}.",
diff --git a/pywikibot/login.py b/pywikibot/login.py
index ac37803..de84d9e 100644
--- a/pywikibot/login.py
+++ b/pywikibot/login.py
@@ -333,7 +333,7 @@
@type sysop: bool
@raises NoUsername: No username is configured for the requested site.
- @raise OAuthImpossible: mwoauth isn't installed
+ @raises OAuthImpossible: mwoauth isn't installed
"""
if isinstance(mwoauth, ImportError):
raise OAuthImpossible('mwoauth is not installed: %s.' % mwoauth)
diff --git a/pywikibot/page.py b/pywikibot/page.py
index 9bb1798..180ad39 100644
--- a/pywikibot/page.py
+++ b/pywikibot/page.py
@@ -3,6 +3,7 @@
Objects representing various types of MediaWiki, including Wikibase, pages.
This module also includes objects:
+
* Property: a type of semantic data.
* Claim: an instance of a semantic assertion.
* Revision: a single change to a wiki page.
@@ -2024,8 +2025,8 @@
"""Iterate templates used on this Page.
@return: a generator that yields a tuple for each use of a template
- in the page, with the template Page as the first entry and a list of
- parameters as the second entry.
+ in the page, with the template Page as the first entry and a list of
+ parameters as the second entry.
"""
# WARNING: may not return all templates used in particularly
# intricate cases such as template substitution
@@ -2270,7 +2271,7 @@
def getFileVersionHistory(self):
"""Return the file's version history.
- @return: A list of dictionaries with the following keys::
+ @return: A list of dictionaries with the following keys:
[comment, sha1, url, timestamp, metadata,
height, width, mime, user, descriptionurl, size]
@@ -3092,10 +3093,10 @@
@kwarg entity_type: Wikibase entity type
@type entity_type: str ('item' or 'property')
- @raise TypeError: incorrect use of parameters
- @raise ValueError: incorrect namespace
- @raise pywikibot.Error: title parsing problems
- @raise NotImplementedError: the entity type is not supported
+ @raises TypeError: incorrect use of parameters
+ @raises ValueError: incorrect namespace
+ @raises pywikibot.Error: title parsing problems
+ @raises NotImplementedError: the entity type is not supported
"""
if not isinstance(site, pywikibot.site.DataSite):
raise TypeError("site must be a pywikibot.site.DataSite object")
@@ -3894,6 +3895,7 @@
Make the item redirect to another item.
You need to define an extra argument to make this work, like save=True
+
@param target_page: target of the redirect, this argument is required.
@type target_page: pywikibot.Item or string
@param force: if true, it sets the redirect target even the page
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 7ba0b6b..a82da93 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -926,9 +926,9 @@
"""
Return the site for a corresponding interwiki prefix.
- @raise SiteDefinitionError: if the url given in the interwiki table
+ @raises SiteDefinitionError: if the url given in the interwiki table
doesn't match any of the existing families.
- @raise KeyError: if the prefix is not an interwiki prefix.
+ @raises KeyError: if the prefix is not an interwiki prefix.
"""
return self._interwikimap[prefix].site
@@ -944,7 +944,7 @@
@type site: L{BaseSite}
@return: The interwiki prefixes
@rtype: list (guaranteed to be not empty)
- @raise KeyError: if there is no interwiki prefix for that site.
+ @raises KeyError: if there is no interwiki prefix for that site.
"""
assert site is not None, 'Site must not be None'
prefixes = set()
@@ -963,9 +963,9 @@
link. So if that link also contains an interwiki link it does follow
it as long as it's a local link.
- @raise SiteDefinitionError: if the url given in the interwiki table
+ @raises SiteDefinitionError: if the url given in the interwiki table
doesn't match any of the existing families.
- @raise KeyError: if the prefix is not an interwiki prefix.
+ @raises KeyError: if the prefix is not an interwiki prefix.
"""
return self._interwikimap[prefix].local
@@ -1569,7 +1569,7 @@
@type expiry: int/float (days), L{datetime.timedelta}, False (never)
@return: The gathered property
@rtype: various
- @raise KeyError: If the key is not a valid siteinfo property and the
+ @raises KeyError: If the key is not a valid siteinfo property and the
get_default option is set to False.
@see: L{_get_siteinfo}
"""
@@ -2012,7 +2012,7 @@
Also logs out of the global account if linked to the user.
- @raise APIError: Logout is not available when OAuth enabled.
+ @raises APIError: Logout is not available when OAuth enabled.
"""
if self.is_oauth_token_available():
pywikibot.warning('Using OAuth suppresses logout function')
@@ -2899,10 +2899,10 @@
@return: redirect target of page
@rtype: BasePage
- @raise IsNotRedirectPage: page is not a redirect
- @raise RuntimeError: no redirects found
- @raise CircularRedirect: page is a circular redirect
- @raise InterwikiRedirectPage: the redirect target is
+ @raises IsNotRedirectPage: page is not a redirect
+ @raises RuntimeError: no redirects found
+ @raises CircularRedirect: page is a circular redirect
+ @raises InterwikiRedirectPage: the redirect target is
on another site
"""
if not self.page_isredirect(page):
--
To view, visit https://gerrit.wikimedia.org/r/238039
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I3c5b6bff67786c08797a8fe6df34a43e62eb952d
Gerrit-PatchSet: 5
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: [tox] shorten lines beneath 100 chars.
......................................................................
[tox] shorten lines beneath 100 chars.
Change-Id: Idf2e45db2f83e9ea1a473b25430bc49b8a829e1d
---
M pywikibot/comms/rcstream.py
M scripts/harvest_template.py
M scripts/interwiki.py
3 files changed, 33 insertions(+), 16 deletions(-)
Approvals:
XZise: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/comms/rcstream.py b/pywikibot/comms/rcstream.py
index 68ebded..2cce8ae 100644
--- a/pywikibot/comms/rcstream.py
+++ b/pywikibot/comms/rcstream.py
@@ -163,7 +163,7 @@
@raises ImportError
- [1] https://github.com/wikimedia/mediawiki/blob/master/includes/rcfeed/MachineR…
+ [1]: See mediawiki/includes/rcfeed/MachineReadableRCFeedFormatter.php
"""
try:
diff --git a/scripts/harvest_template.py b/scripts/harvest_template.py
index ec5fc3b..c1ea88c 100755
--- a/scripts/harvest_template.py
+++ b/scripts/harvest_template.py
@@ -85,7 +85,8 @@
pywikibot.error(u'Template %s does not exist.' % temp.title())
exit()
- pywikibot.output('Finding redirects...') # Put some output here since it can take a while
+ # Put some output here since it can take a while
+ pywikibot.output('Finding redirects...')
if temp.isRedirectPage():
temp = temp.getRedirectTarget()
titles = [page.title(withNamespace=False)
@@ -143,7 +144,9 @@
template = pywikibot.Page(page.site, template,
ns=10).title(withNamespace=False)
except pywikibot.exceptions.InvalidTitle:
- pywikibot.error(u"Failed parsing template; '%s' should be the template name." % template)
+ pywikibot.error(
+ "Failed parsing template; '%s' should be the template name."
+ % template)
continue
# We found the template we were looking for
if template in self.templateTitles:
@@ -191,11 +194,15 @@
if image.isRedirectPage():
image = pywikibot.FilePage(image.getRedirectTarget())
if not image.exists():
- pywikibot.output('[[%s]] doesn\'t exist so I can\'t link to it' % (image.title(),))
+ pywikibot.output(
+ '[[%s]] doesn\'t exist so I can\'t link to it'
+ % (image.title(),))
continue
claim.setTarget(image)
else:
- pywikibot.output("%s is not a supported datatype." % claim.type)
+ pywikibot.output(
+ '%s is not a supported datatype.'
+ % claim.type)
continue
pywikibot.output('Adding %s --> %s'
diff --git a/scripts/interwiki.py b/scripts/interwiki.py
index 58c46b1..5e51723 100755
--- a/scripts/interwiki.py
+++ b/scripts/interwiki.py
@@ -358,7 +358,8 @@
import pywikibot
-from pywikibot import config, i18n, pagegenerators, textlib, interwiki_graph, titletranslate
+from pywikibot import config, i18n, pagegenerators, textlib, interwiki_graph
+from pywikibot import titletranslate
from pywikibot.bot import ListOption, StandardOption
from pywikibot.tools import first_upper
@@ -584,7 +585,8 @@
elif arg.startswith('-neverlink:'):
self.neverlink += arg[11:].split(",")
elif arg.startswith('-ignore:'):
- self.ignore += [pywikibot.Page(pywikibot.Site(), p) for p in arg[8:].split(",")]
+ self.ignore += [pywikibot.Page(pywikibot.Site(), p)
+ for p in arg[8:].split(',')]
elif arg.startswith('-ignorefile:'):
ignorefile = arg[12:]
ignorePageGen = pagegenerators.TextfilePageGenerator(ignorefile)
@@ -1055,7 +1057,8 @@
return False
if globalvar.autonomous:
pywikibot.output(
-u"NOTE: Ignoring link from page %s in namespace %i to page %s in namespace %i."
+ 'NOTE: Ignoring link from page %s in namespace %i to page '
+ '%s in namespace %i.'
% (linkingPage, linkingPage.namespace(), linkedPage,
linkedPage.namespace()))
# Fill up foundIn, so that we will not write this notice
@@ -1065,14 +1068,16 @@
preferredPage = self.getFoundInCorrectNamespace(linkedPage.site)
if preferredPage:
pywikibot.output(
-u"NOTE: Ignoring link from page %s in namespace %i to page %s in namespace %i "
-u"because page %s in the correct namespace has already been found."
+ 'NOTE: Ignoring link from page %s in namespace %i to '
+ 'page %s in namespace %i because page %s in the '
+ 'correct namespace has already been found.'
% (linkingPage, linkingPage.namespace(), linkedPage,
linkedPage.namespace(), preferredPage))
return True
else:
choice = pywikibot.input_choice(
-u'WARNING: %s is in namespace %i, but %s is in namespace %i. Follow it anyway?'
+ 'WARNING: %s is in namespace %i, but %s is in '
+ 'namespace %i. Follow it anyway?'
% (self.originPage, self.originPage.namespace(),
linkedPage, linkedPage.namespace()),
[('Yes', 'y'), ('No', 'n'),
@@ -1458,8 +1463,9 @@
linkedPage = pywikibot.Page(link)
if globalvar.hintsareright:
if linkedPage.site in self.hintedsites:
- pywikibot.output(u"NOTE: %s: %s extra interwiki on hinted site ignored %s"
- % (self.originPage, page, linkedPage))
+ pywikibot.output(
+ 'NOTE: %s: %s extra interwiki on hinted site ignored %s'
+ % (self.originPage, page, linkedPage))
break
if not self.skipPage(page, linkedPage, counter):
if globalvar.followinterwiki or page == self.originPage:
@@ -1912,7 +1918,8 @@
):
new[rmsite] = rmPage
pywikibot.output(
- u"WARNING: %s is either deleted or has a mismatching disambiguation state."
+ 'WARNING: %s is either deleted or has a mismatching '
+ 'disambiguation state.'
% rmPage)
# Re-Check what needs to get done
mods, mcomment, adding, removing, modifying = compareLanguages(old,
@@ -2196,7 +2203,9 @@
except KeyError:
pass
if loc is not None and loc in page.title():
- pywikibot.output(u'Skipping: %s is a templates subpage' % page.title())
+ pywikibot.output(
+ 'Skipping: %s is a templates subpage'
+ % page.title())
continue
break
@@ -2265,7 +2274,8 @@
except pywikibot.ServerError:
# Could not extract allpages special page?
pywikibot.output(
- u'ERROR: could not retrieve more pages. Will try again in %d seconds'
+ 'ERROR: could not retrieve more pages. '
+ 'Will try again in %d seconds'
% timeout)
time.sleep(timeout)
timeout *= 2
--
To view, visit https://gerrit.wikimedia.org/r/231896
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Idf2e45db2f83e9ea1a473b25430bc49b8a829e1d
Gerrit-PatchSet: 5
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: Fix spelling
......................................................................
Fix spelling
Change-Id: Iaa40d502a2ca03a0fd802a0735d2d7fc76446dd4
---
M scripts/pagefromfile.py
1 file changed, 1 insertion(+), 1 deletion(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/pagefromfile.py b/scripts/pagefromfile.py
index 1b84686..4eb65a8 100755
--- a/scripts/pagefromfile.py
+++ b/scripts/pagefromfile.py
@@ -40,7 +40,7 @@
-autosummary Use MediaWikis autosummary when creating a new page,
overrides -summary in this case
-minor set minor edit flag on page edits
--showdiff show difference between pag and page to upload; it forces
+-showdiff show difference between page and page to upload; it forces
-always=False; default to False.
If the page to be uploaded already exists:
--
To view, visit https://gerrit.wikimedia.org/r/238400
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Iaa40d502a2ca03a0fd802a0735d2d7fc76446dd4
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: Changes are wrongly detected in the last langlink
......................................................................
Changes are wrongly detected in the last langlink
This was first detected in cosmetic_changes.py, where pages not needing any
cosmetic changes were still being updated, and the supposed changed was an
unchanged langlink (the last one). Then I realized this also happens in
interwiki.py, but only when changes are supposed to be made to the page.
After checking textlib.py, I noticed that removeLanguageLinks() returns a
stripped version of the text whereas replaceLanguageLinks() does not. So
the actual difference is related to blank space.
This explains why cosmetic_changes.py detects a change and updates the page,
but interwiki.py does not: interwiki.py does not rely in differences in the
final text to make an update, but when it does need to make an update, it
outputs that diff.
This patch strips the text in replaceLanguageLinks() before returning if the
original text is stripped itself.
Change-Id: Icd7252be8dbccf3fb04a4b4a465f6b057e3a8e3a
---
M pywikibot/textlib.py
1 file changed, 9 insertions(+), 2 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/textlib.py b/pywikibot/textlib.py
index 5c13101..72d68b6 100644
--- a/pywikibot/textlib.py
+++ b/pywikibot/textlib.py
@@ -38,7 +38,11 @@
from pywikibot import config2 as config
from pywikibot.exceptions import InvalidTitle
from pywikibot.family import Family
-from pywikibot.tools import OrderedDict, DeprecatedRegex
+from pywikibot.tools import (
+ DeprecatedRegex,
+ OrderedDict,
+ issue_deprecation_warning
+)
# cache for replaceExcept to avoid recompile or regexes each call
_regex_cache = {}
@@ -843,6 +847,9 @@
cseparator = site.family.category_text_separator
separatorstripped = separator.strip()
cseparatorstripped = cseparator.strip()
+ do_not_strip = oldtext.strip() != oldtext
+ if do_not_strip:
+ issue_deprecation_warning('Using unstripped text', 'stripped text', 2)
if addOnly:
s2 = oldtext
else:
@@ -910,7 +917,7 @@
newtext = s2.replace(marker, '').strip() + separator + s
else:
newtext = s2.replace(marker, '')
- return newtext
+ return newtext if do_not_strip else newtext.strip()
def interwikiFormat(links, insite=None):
--
To view, visit https://gerrit.wikimedia.org/r/238149
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Icd7252be8dbccf3fb04a4b4a465f6b057e3a8e3a
Gerrit-PatchSet: 7
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Malafaya <malafaya(a)clix.pt>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Malafaya <malafaya(a)clix.pt>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: [FEAT] Support writing to any compressed file
......................................................................
[FEAT] Support writing to any compressed file
Instead of just opening the file to read it, this also allows to write them. To
have a similar naming scheme like `open` it renames the function to allow
changing the signature.
It now by default uses the file extension and not the magic number when using
the new function. And the category script now uses the new function.
Bug: T112373
Change-Id: I24f5477c2488387cd7811e4c38859323720aef88
---
M pywikibot/tools/__init__.py
M pywikibot/xmlreader.py
M scripts/category.py
M tests/tools_tests.py
4 files changed, 141 insertions(+), 38 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/tools/__init__.py b/pywikibot/tools/__init__.py
index 5776222..537d2dc 100644
--- a/pywikibot/tools/__init__.py
+++ b/pywikibot/tools/__init__.py
@@ -884,23 +884,32 @@
setattr(self._wrapped, name, value)
-def open_compressed(filename, use_extension=False):
+def open_archive(filename, mode='rb', use_extension=True):
"""
Open a file and uncompress it if needed.
This function supports bzip2, gzip and 7zip as compression containers. It
uses the packages available in the standard library for bzip2 and gzip so
they are always available. 7zip is only available when a 7za program is
- available.
+ available and only supports reading from it.
The compression is either selected via the magic number or file ending.
@param filename: The filename.
@type filename: str
@param use_extension: Use the file extension instead of the magic number
- to determine the type of compression (default False).
+ to determine the type of compression (default True). Must be True when
+ writing or appending.
@type use_extension: bool
- @raises ValueError: When 7za is not available.
+ @param mode: The mode in which the file should be opened. It may either be
+ 'r', 'rb', 'a', 'ab', 'w' or 'wb'. All modes open the file in binary
+ mode. It defaults to 'rb'.
+ @type mode: string
+ @raises ValueError: When 7za is not available or the opening mode is unknown
+ or it tries to write a 7z archive.
+ @raises FileNotFoundError: When the filename doesn't exist and it tries
+ to read from it or it tries to determine the compression algorithm (or
+ IOError on Python 2).
@raises OSError: When it's not a 7z archive but the file extension is 7z.
It is also raised by bz2 when its content is invalid. gzip does not
immediately raise that error but only on reading it.
@@ -916,11 +925,18 @@
else:
return wrapped
+ if mode in ('r', 'a', 'w'):
+ mode += 'b'
+ elif mode not in ('rb', 'ab', 'wb'):
+ raise ValueError('Invalid mode: "{0}"'.format(mode))
+
if use_extension:
# if '.' not in filename, it'll be 1 character long but otherwise
# contain the period
extension = filename[filename.rfind('.'):][1:]
else:
+ if mode != 'rb':
+ raise ValueError('Magic number detection only when reading')
with open(filename, 'rb') as f:
magic_number = f.read(8)
if magic_number.startswith(b'BZh'):
@@ -933,10 +949,13 @@
extension = ''
if extension == 'bz2':
- return wrap(bz2.BZ2File(filename), 1)
+ return wrap(bz2.BZ2File(filename, mode), 1)
elif extension == 'gz':
- return wrap(gzip.open(filename), 0)
+ return wrap(gzip.open(filename, mode), 0)
elif extension == '7z':
+ if mode != 'rb':
+ raise NotImplementedError('It is not possible to write a 7z file.')
+
try:
process = subprocess.Popen(['7za', 'e', '-bd', '-so', filename],
stdout=subprocess.PIPE,
@@ -1496,3 +1515,9 @@
if self._deprecated[attr][1]:
return self._deprecated[attr][1]
return getattr(self._module, attr)
+
+
+@deprecated('open_archive()')
+def open_compressed(filename, use_extension=False):
+ """DEPRECATED: Open a file and uncompress it if needed."""
+ return open_archive(filename, use_extension=use_extension)
diff --git a/pywikibot/xmlreader.py b/pywikibot/xmlreader.py
index 86185cf..0846acd 100644
--- a/pywikibot/xmlreader.py
+++ b/pywikibot/xmlreader.py
@@ -23,7 +23,7 @@
from xml.etree.cElementTree import iterparse
import xml.sax
-from pywikibot.tools import open_compressed
+from pywikibot.tools import open_archive
def parseRestrictions(restrictions):
@@ -118,7 +118,7 @@
def parse(self):
"""Generator using cElementTree iterparse function."""
- with open_compressed(self.filename) as source:
+ with open_archive(self.filename) as source:
# iterparse's event must be a str but they are unicode with
# unicode_literals in Python 2
context = iterparse(source, events=(str('start'), str('end'),
diff --git a/scripts/category.py b/scripts/category.py
index 2bd8973..537a1ab 100755
--- a/scripts/category.py
+++ b/scripts/category.py
@@ -122,7 +122,6 @@
import os
import re
import pickle
-import bz2
import sys
import pywikibot
@@ -132,7 +131,7 @@
MultipleSitesBot, IntegerOption, StandardOption, ContextOption,
)
from pywikibot.tools import (
- deprecated_args, deprecated, ModuleDeprecationWrapper
+ deprecated_args, deprecated, ModuleDeprecationWrapper, open_archive
)
if sys.version_info[0] > 2:
@@ -183,11 +182,10 @@
def _load(self):
if not self.is_loaded:
try:
- f = bz2.BZ2File(self.filename, 'r')
pywikibot.output(u'Reading dump from %s'
% config.shortpath(self.filename))
- databases = pickle.load(f)
- f.close()
+ with open_archive(self.filename, 'rb') as f:
+ databases = pickle.load(f)
# keys are categories, values are 2-tuples with lists as
# entries.
self.catContentDB = databases['catContentDB']
@@ -265,17 +263,16 @@
if self.is_loaded and (self.catContentDB or self.superclassDB):
pywikibot.output(u'Dumping to %s, please wait...'
% config.shortpath(filename))
- f = bz2.BZ2File(filename, 'w')
databases = {
'catContentDB': self.catContentDB,
'superclassDB': self.superclassDB
}
# store dump to disk in binary format
- try:
- pickle.dump(databases, f, protocol=config.pickle_protocol)
- except pickle.PicklingError:
- pass
- f.close()
+ with open_archive(filename, 'wb') as f:
+ try:
+ pickle.dump(databases, f, protocol=config.pickle_protocol)
+ except pickle.PicklingError:
+ pass
else:
try:
os.remove(filename)
diff --git a/tests/tools_tests.py b/tests/tools_tests.py
index dbb8df8..c19e354 100644
--- a/tests/tools_tests.py
+++ b/tests/tools_tests.py
@@ -14,11 +14,12 @@
import os.path
import subprocess
import sys
+import tempfile
from pywikibot import tools
from tests import _data_dir
-from tests.aspects import unittest, TestCase
+from tests.aspects import unittest, DeprecationTestCase, TestCase
from tests.utils import expected_failure_if
_xml_data_dir = os.path.join(_data_dir, 'xml')
@@ -70,15 +71,15 @@
self.assertTrue(wrapper.closed)
-class OpenCompressedTestCase(TestCase):
+class OpenArchiveTestCase(TestCase):
"""
Unit test class for tools.
- The tests for open_compressed requires that article-pyrus.xml* contain all
+ The tests for open_archive requires that article-pyrus.xml* contain all
the same content after extraction. The content itself is not important.
The file article-pyrus.xml_invalid.7z is not a valid 7z file and
- open_compressed will fail extracting it using 7za.
+ open_archive will fail extracting it using 7za.
"""
net = False
@@ -86,38 +87,118 @@
@classmethod
def setUpClass(cls):
"""Define base_file and original_content."""
- super(OpenCompressedTestCase, cls).setUpClass()
+ super(OpenArchiveTestCase, cls).setUpClass()
cls.base_file = os.path.join(_xml_data_dir, 'article-pyrus.xml')
with open(cls.base_file, 'rb') as f:
cls.original_content = f.read()
- @staticmethod
- def _get_content(*args):
- """Use open_compressed and return content using a with-statement."""
- with tools.open_compressed(*args) as f:
+ def _get_content(self, *args, **kwargs):
+ """Use open_archive and return content using a with-statement."""
+ with tools.open_archive(*args, **kwargs) as f:
return f.read()
- def test_open_compressed_normal(self):
- """Test open_compressed with no compression in the standard library."""
+ def test_open_archive_normal(self):
+ """Test open_archive with no compression in the standard library."""
self.assertEqual(self._get_content(self.base_file), self.original_content)
- def test_open_compressed_bz2(self):
- """Test open_compressed with bz2 compressor in the standard library."""
+ def test_open_archive_bz2(self):
+ """Test open_archive with bz2 compressor in the standard library."""
self.assertEqual(self._get_content(self.base_file + '.bz2'), self.original_content)
- self.assertEqual(self._get_content(self.base_file + '.bz2', True), self.original_content)
+ self.assertEqual(self._get_content(self.base_file + '.bz2', use_extension=False),
+ self.original_content)
- def test_open_compressed_gz(self):
- """Test open_compressed with gz compressor in the standard library."""
+ def test_open_archive_gz(self):
+ """Test open_archive with gz compressor in the standard library."""
self.assertEqual(self._get_content(self.base_file + '.gz'), self.original_content)
- def test_open_compressed_7z(self):
- """Test open_compressed with 7za if installed."""
+ def test_open_archive_7z(self):
+ """Test open_archive with 7za if installed."""
try:
subprocess.Popen(['7za'], stdout=subprocess.PIPE).stdout.close()
except OSError:
raise unittest.SkipTest('7za not installed')
self.assertEqual(self._get_content(self.base_file + '.7z'), self.original_content)
- self.assertRaises(OSError, self._get_content, self.base_file + '_invalid.7z', True)
+ self.assertRaises(OSError, self._get_content, self.base_file + '_invalid.7z',
+ use_extension=True)
+
+
+class OpenCompressedTestCase(OpenArchiveTestCase, DeprecationTestCase):
+
+ """Test opening files with the deprecated open_compressed."""
+
+ net = False
+
+ def _get_content(self, *args, **kwargs):
+ """Use open_compressed and return content using a with-statement."""
+ # open_archive default is True, so if it's False it's not the default
+ # so use the non-default of open_compressed (which is True)
+ if kwargs.get('use_extension') is False:
+ kwargs['use_extension'] = True
+
+ with tools.open_compressed(*args, **kwargs) as f:
+ content = f.read()
+ self.assertOneDeprecation(self.INSTEAD)
+ return content
+
+
+class OpenArchiveWriteTestCase(TestCase):
+
+ """Test writing with open_archive."""
+
+ net = False
+
+ @classmethod
+ def setUpClass(cls):
+ """Define base_file and original_content."""
+ super(OpenArchiveWriteTestCase, cls).setUpClass()
+ cls.base_file = os.path.join(_xml_data_dir, 'article-pyrus.xml')
+ with open(cls.base_file, 'rb') as f:
+ cls.original_content = f.read()
+
+ def _write_content(self, suffix):
+ try:
+ fn = tempfile.mkstemp(suffix)[1]
+ with tools.open_archive(fn, 'wb') as f:
+ f.write(self.original_content)
+ with tools.open_archive(fn, 'rb') as f:
+ self.assertEqual(f.read(), self.original_content)
+ with open(fn, 'rb') as f:
+ return f.read()
+ finally:
+ os.remove(fn)
+
+ def test_invalid_modes(self):
+ """Test various invalid mode configurations."""
+ self.assertRaises(ValueError, tools.open_archive,
+ '/dev/null', 'ra') # two modes besides
+ self.assertRaises(ValueError, tools.open_archive,
+ '/dev/null', 'rt') # text mode
+ self.assertRaises(ValueError, tools.open_archive,
+ '/dev/null', 'br') # binary at front
+ self.assertRaises(ValueError, tools.open_archive,
+ '/dev/null', 'wb', False) # writing without extension
+
+ def test_binary_mode(self):
+ """Test that it uses binary mode."""
+ with tools.open_archive(self.base_file, 'r') as f:
+ self.assertEqual(f.mode, 'rb')
+ self.assertIsInstance(f.read(), bytes)
+
+ def test_write_archive_bz2(self):
+ """Test writing a bz2 archive."""
+ content = self._write_content('.bz2')
+ with open(self.base_file + '.bz2', 'rb') as f:
+ self.assertEqual(content, f.read())
+
+ def test_write_archive_gz(self):
+ """Test writing a gz archive."""
+ content = self._write_content('.gz')
+ self.assertEqual(content[:3], b'\x1F\x8B\x08')
+
+ def test_write_archive_7z(self):
+ """Test writing an archive as a 7z archive."""
+ self.assertRaises(NotImplementedError, tools.open_archive,
+ '/dev/null.7z', mode='wb')
class MergeUniqueDicts(TestCase):
--
To view, visit https://gerrit.wikimedia.org/r/238315
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I24f5477c2488387cd7811e4c38859323720aef88
Gerrit-PatchSet: 4
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>