jenkins-bot has submitted this change and it was merged.
Change subject: docs: only apply magic to scripts
......................................................................
docs: only apply magic to scripts
Change-Id: I651ccd9a621b59280923ab6c7f2fda4bd06708d7
---
M docs/conf.py
1 file changed, 5 insertions(+), 0 deletions(-)
Approvals:
XZise: Looks good to me, approved
jenkins-bot: Verified
diff --git a/docs/conf.py b/docs/conf.py
index 7d24345..917ff79 100644
--- a/docs/conf.py
+++ b/docs/conf.py
@@ -248,6 +248,11 @@
def pywikibot_script_docstring_fixups(
app, what, name, obj, options, lines):
"""Pywikibot specific conversions."""
+ if what != "module":
+ return
+
+ if os.path.sep + "scripts" + os.path.sep not in obj.__file__:
+ return
for index, line in enumerate(lines):
if line in ('¶ms;', '&pagegenerators_help;'):
lines[index] = ('This script supports use of '
--
To view, visit https://gerrit.wikimedia.org/r/238011
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I651ccd9a621b59280923ab6c7f2fda4bd06708d7
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: [BREAK][FEAT]lonelypages: Simplify orphan config
......................................................................
[BREAK][FEAT]lonelypages: Simplify orphan config
Instead of storing the orphan templates separately (one mapping for the
used template and one regex for the aliases) it's dynamically creating
both depending on the given aliases and names. It'll also automatically
choose all template namespace prefixes in the given language. The regex
to match the template will be also compiled only once and not for each
page.
Change-Id: Ie9c86c196ef33ff7bb336bb2bac9e688b41938a3
---
M scripts/lonelypages.py
1 file changed, 69 insertions(+), 42 deletions(-)
Approvals:
Merlijn van Deen: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/lonelypages.py b/scripts/lonelypages.py
index d3ec6e4..3104941 100755
--- a/scripts/lonelypages.py
+++ b/scripts/lonelypages.py
@@ -33,7 +33,7 @@
#
# (C) Pietrodn, it.wiki 2006-2007
# (C) Filnik, it.wiki 2007
-# (C) Pywikibot team, 2008-2014
+# (C) Pywikibot team, 2008-2015
#
# Distributed under the terms of the MIT license.
#
@@ -46,7 +46,8 @@
import sys
import pywikibot
-from pywikibot import i18n, pagegenerators, Bot
+from pywikibot import i18n, pagegenerators
+from pywikibot.bot import suggest_help, SingleSiteBot
# This is required for the text that is shown when you run this script
# with the parameter -help.
@@ -54,28 +55,51 @@
'¶ms;': pagegenerators.parameterHelp,
}
-template = {
- 'ar': u'{{يتيمة|تاريخ={{نسخ:اسم_شهر}} {{نسخ:عام}}}}',
- 'ca': u'{{Orfe|date={{subst:CURRENTMONTHNAME}} {{subst:CURRENTYEAR}}}}',
- 'en': u'{{Orphan|date={{subst:CURRENTMONTHNAME}} {{subst:CURRENTYEAR}}}}',
- 'it': u'{{O||mese={{subst:CURRENTMONTHNAME}} {{subst:CURRENTYEAR}}}}',
- 'ja': u'{{孤立|{{subst:DATE}}}}',
- 'zh': u'{{subst:Orphan/auto}}',
-}
-# Use regex to prevent to put the same template twice!
-exception_regex = {
- 'ar': [u'\\{\\{(?:قالب:|)(يتيمة)[\\|\\}]'],
- 'ca': [r'\{\{(?:template:|)(orfe)[\|\}]'],
- 'en': [r'\{\{(?:template:|)(orphan)[\|\}]',
- r'\{\{(?:template:|)(wi)[\|\}]'],
- 'it': [r'\{\{(?:template:|)(o|a)[\|\}]'],
- 'ja': [u'\\{\\{(?:template:|)(孤立)[\\|\\}]'],
- 'zh': [r'\{\{(?:template:|)(orphan)[\|\}]'],
+class OrphanTemplate(object):
+
+ """The orphan template configuration."""
+
+ def __init__(self, site, name, parameters, aliases=None, subst=False):
+ self._name = name
+ if not aliases:
+ aliases = []
+ elif not subst:
+ aliases = list(aliases) + [name]
+ else:
+ name = 'subst:' + name
+ if parameters:
+ name += '|' + parameters
+ self.template = '{{' + name + '}}'
+ self._names = frozenset(aliases)
+
+ template_ns = site.namespaces[10]
+ # TODO: Add redirects to self.names too
+ if not pywikibot.Page(site, self._name, template_ns.id).exists():
+ raise ValueError('Orphan template "{0}" does not exist on '
+ '"{1}".'.format(self._name, site))
+ for name in self._names:
+ if not pywikibot.Page(site, name, template_ns.id).exists():
+ pywikibot.warning('Orphan template alias "{0}" does not exist '
+ 'on "{1}"'.format(name, site))
+ self.regex = re.compile(
+ r'\{\{(?:' + ':|'.join(template_ns) + '|)(' +
+ '|'.join(re.escape(name) for name in self._names) +
+ r')[\|\}]', re.I)
+
+
+# The orphan template names in the different languages.
+_templates = {
+ 'ar': ('ﻲﺘﻴﻣﺓ', 'ﺕﺍﺮﻴﺧ={{ﻦﺴﺧ:ﺎﺴﻣ_ﺶﻫﺭ}} {{ﻦﺴﺧ:ﻉﺎﻣ}}'),
+ 'ca': ('Orfe', 'date={{subst:CURRENTMONTHNAME}} {{subst:CURRENTYEAR}}'),
+ 'en': ('Orphan', 'date={{subst:CURRENTMONTHNAME}} {{subst:CURRENTYEAR}}', ['wi']),
+ 'it': ('O', '||mese={{subst:CURRENTMONTHNAME}} {{subst:CURRENTYEAR}}', ['a']),
+ 'ja': ('孤立', '{{subst:DATE}}'),
+ 'zh': ('Orphan/auto', '', ['orphan'], True),
}
-class LonelyPagesBot(Bot):
+class LonelyPagesBot(SingleSiteBot):
"""Orphan page tagging bot."""
@@ -88,7 +112,6 @@
self.generator = generator
# Take the configurations according to our project
- self.site = pywikibot.Site()
if self.getOption('enablePage'):
self.options['enablePage'] = pywikibot.Page(
self.site, self.getOption('enablePage'))
@@ -96,11 +119,18 @@
self.site, 'lonelypages-comment-add-template')
self.commentdisambig = i18n.twtranslate(
self.site, 'lonelypages-comment-add-disambig-template')
- self.template = i18n.translate(self.site, template)
- self.exception = i18n.translate(self.site, exception_regex)
- if self.template is None or self.exception is None:
- pywikibot.showHelp()
- sys.exit(u'Missing configuration for site %s' % self.site)
+ orphan_template = i18n.translate(self.site, _templates)
+ if orphan_template is not None:
+ try:
+ orphan_template = OrphanTemplate(self.site, *orphan_template)
+ except ValueError as e:
+ orphan_template = e
+ if orphan_template is None or isinstance(orphan_template, ValueError):
+ err_message = 'Missing configuration for site %s' % self.site
+ suggest_help(exception=orphan_template, additional_text=err_message)
+ sys.exit(err_message)
+ else:
+ self._settings = orphan_template
# DisambigPage part
if self.getOption('disambigPage') is not None:
self.disambigpage = pywikibot.Page(self.site, self.getOption('disambigPage'))
@@ -113,6 +143,11 @@
pywikibot.output(u"%s is a redirect, don't use it!"
% self.disambigpage.title())
self.options['disambigPage'] = None
+
+ @property
+ def settings(self):
+ """Return the settings for the configured site."""
+ return self._settings
def enable_page(self):
enable = self.getOption('enablePage')
@@ -136,11 +171,10 @@
if not self.enable_page():
pywikibot.output('The bot is disabled')
return
- # Main Loop
- for page in self.generator:
- self.treat(page)
+ super(LonelyPagesBot, self).run()
def treat(self, page):
+ """Check if page is applicable and not marked and add template then."""
pywikibot.output(u"Checking %s..." % page.title())
if page.isRedirectPage(): # If redirect, skip!
pywikibot.output(u'%s is a redirect! Skip...' % page.title())
@@ -159,18 +193,10 @@
except pywikibot.IsRedirectPage:
pywikibot.output(u"%s is a redirect! Skip..." % page.title())
return
- # I've used a loop in a loop. If I use continue in the second loop,
- # it won't do anything in the first. So let's create a variable to
- # avoid this problem.
- for regexp in self.exception:
- res = re.findall(regexp, oldtxt.lower())
- # Found a template! Let's skip the page!
- if res != []:
- pywikibot.output(
- u'Your regex has found something in %s, skipping...'
- % page.title())
- break
- else:
+ if self.settings.regex.search(oldtxt):
+ pywikibot.output(
+ u'Your regex has found something in %s, skipping...'
+ % page.title())
return
if page.isDisambig() and self.getOption('disambigPage') is not None:
pywikibot.output(u'%s is a disambig page, report..'
@@ -188,7 +214,7 @@
else:
# Ok, the page need the template. Let's put it there!
# Adding the template in the text
- newtxt = u"%s\n%s" % (self.template, oldtxt)
+ newtxt = '%s\n%s' % (self.settings.template, oldtxt)
self.userPut(page, oldtxt, newtxt, summary=self.comment)
@@ -234,5 +260,6 @@
bot = LonelyPagesBot(generator, **options)
bot.run()
+
if __name__ == '__main__':
main()
--
To view, visit https://gerrit.wikimedia.org/r/201446
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ie9c86c196ef33ff7bb336bb2bac9e688b41938a3
Gerrit-PatchSet: 12
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: Eranroz <eranroz89(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: [FEAT] bot: Explain and document Bot classes
......................................................................
[FEAT] bot: Explain and document Bot classes
There are a lot of different `Bot` classes and it might be confusing which
class does what and how they are supposed to be used. This is a first attempt
in explaining them.
Change-Id: I09b4751290e743aa2d7e2526a58f382b31c78ee7
---
M pywikibot/bot.py
1 file changed, 45 insertions(+), 1 deletion(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/bot.py b/pywikibot/bot.py
index 6c334b6..dae9760 100644
--- a/pywikibot/bot.py
+++ b/pywikibot/bot.py
@@ -1,5 +1,49 @@
# -*- coding: utf-8 -*-
-"""User-interface related functions for building bots."""
+"""
+User-interface related functions for building bots.
+
+This module supports several different bot classes which could be used in
+conjunction. Each bot should subclass at least one of these four classes:
+
+* L{BaseBot}: Basic bot class in case where the site is handled differently,
+ like working on two sites in parallel.
+* L{SingleSiteBot}: Bot class which should only be run on a single site. They
+ usually store site specific content and thus can't be easily run when the
+ generator returns a page on another site. It has a property C{site} which
+ can also be changed. If the generator returns a page of a different site
+ it'll skip that page.
+* L{MultipleSitesBot}: Bot class which supports to be run on multiple sites
+ without the need to manually initialize it every time. It is not possible to
+ set the C{site} property and it's deprecated to request it. Instead site of
+ the current page should be used. And out of C{run} that sit isn't defined.
+* L{Bot}: The previous base class which should be avoided. This class is mainly
+ used for bots which work with wikibase or together with an image repository.
+
+Additionally there is the L{CurrentPageBot} class which automatically sets the
+current page to the page treated. It is recommended to use this class and to
+use C{treat_page} instead of C{treat} and C{put_current} instead of C{userPut}.
+It by default subclasses the C{BaseBot} class.
+
+With L{CurrentPageBot} it's possible to subclass one of the following classes to
+filter the pages which are ultimately handled by C{treat_page}:
+
+* L{ExistingPageBot}: Only handle pages which do exist.
+* L{CreatingPageBot}: Only handle pages which do not exist.
+* L{RedirectPageBot}: Only handle pages which are redirect pages.
+* L{NoRedirectPageBot}: Only handle pages which are not redirect pages.
+* L{FollowRedirectPageBot}: If the generator returns a redirect page it'll
+ follow the redirect and instead work on the redirected class.
+
+It is possible to combine filters by subclassing multiple of them. They are
+new-style classes so when a class is first subclassing L{ExistingPageBot} and
+then L{FollowRedirectPageBot} it will also work on pages which do not exist when
+a redirect pointed to that. If the order is inversed it'll first follow them and
+then check whether they exist.
+
+Additionally there is the L{AutomaticTWSummaryBot} which subclasses
+L{CurrentPageBot} and automatically defines the summary when C{put_current} is
+used.
+"""
#
# (C) Pywikibot team, 2008-2015
#
--
To view, visit https://gerrit.wikimedia.org/r/237990
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I09b4751290e743aa2d7e2526a58f382b31c78ee7
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: [FEAT] upload: Get stash info before continuing
......................................................................
[FEAT] upload: Get stash info before continuing
If the response does contain warnings it won't contain the offset for the
continuation (T73737). So it requests the stash info for the given file key to
determine the offset. This also allows to properly reuse the file key as it
will create a new file key instance when the offset is 0. It also prevents
that the API shows an error if the offset is neither 0 nor the actual size
uploaded.
And if the offset is given it can verify that it matches the stashed data. It
also supports comparing the SHA1 of the already uploaded part to minimize the
chance that a different file is uploaded for continuation. It now also verifies
that the offset is not larger than the file size. In case the file is large and
doing an SHA1 would be time consuming it's possible to not verify them.
The offset is not present if the warning occurred in the first chunk which
would have caused that the first chunk is discarded when continuing to upload.
To do that the file key must be defined before it issues the warnings.
Also when uploading in the unchunked mode and a file key is provided it did
upload the complete file even though the offset was set to `False` to indicate
the file is completely uploaded.
One test is an expected failure due to T112416.
Change-Id: I610a0744d74dc5a2523d96eabde818f76dd6c5e8
---
M pywikibot/site.py
M tests/upload_tests.py
2 files changed, 221 insertions(+), 19 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/site.py b/pywikibot/site.py
index 8cb66a8..9e7dde9 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -16,6 +16,7 @@
#
import datetime
+import hashlib
import itertools
import os
import re
@@ -5177,10 +5178,20 @@
raise
return self._uploaddisabled
+ def stash_info(self, file_key, props=False):
+ """Get the stash info for a given file key."""
+ if not props:
+ props = False
+ req = self._simple_request(
+ action='query', prop='stashimageinfo', siifilekey=file_key,
+ siiprop=props)
+ return req.submit()['query']['stashimageinfo'][0]
+
@deprecate_arg('imagepage', 'filepage')
def upload(self, filepage, source_filename=None, source_url=None,
comment=None, text=None, watch=False, ignore_warnings=False,
- chunk_size=0, _file_key=None, _offset=0, report_success=None):
+ chunk_size=0, _file_key=None, _offset=0, _verify_stash=None,
+ report_success=None):
"""Upload a file to the wiki.
Either source_filename or source_url, but not both, must be provided.
@@ -5215,8 +5226,15 @@
@type _file_key: str or None
@param _offset: When file_key is not None this can be an integer to
continue a previously canceled chunked upload. If False it treats
- that as a finished upload. By default starts at 0.
+ that as a finished upload. If True it requests the stash info from
+ the server to determine the offset. By default starts at 0.
@type _offset: int or bool
+ @param _verify_stash: Requests the SHA1 and file size uploaded and
+ compares it to the local file. Also verifies that _offset is
+ matching the file size if the _offset is an int. If _offset is False
+ if verifies that the file size match with the local file. If None
+ it'll verifies the stash when a file key and offset is given.
+ @type _verify_stash: bool or None
@param report_success: If the upload was successful it'll print a
success message and if ignore_warnings is set to False it'll
raise an UploadWarning if a warning occurred. If it's None (default)
@@ -5231,7 +5249,7 @@
api.UploadWarning(
warning,
upload_warnings.get(warning, '%(msg)s') % {'msg': data},
- _file_key, response.get('offset', 0))
+ _file_key, response['offset'])
for warning, data in response['warnings'].items()]
upload_warnings = {
@@ -5287,7 +5305,76 @@
token = self.tokens['edit']
result = None
file_page_title = filepage.title(withNamespace=False)
- if _file_key and _offset is False:
+ file_size = None
+ offset = _offset
+ # make sure file actually exists
+ if source_filename:
+ if os.path.isfile(source_filename):
+ file_size = os.path.getsize(source_filename)
+ elif offset is not False:
+ raise ValueError("File '%s' does not exist."
+ % source_filename)
+
+ if source_filename and _file_key:
+ assert offset is False or file_size is not None
+ if _verify_stash is None:
+ _verify_stash = True
+ if (offset is not False and offset is not True and
+ offset > file_size):
+ raise ValueError(
+ 'For the file key "{0}" the offset was set to {1} '
+ 'while the file is only {2} bytes large.'.format(
+ _file_key, offset, file_size))
+
+ if _verify_stash or offset is True:
+ if not _file_key:
+ raise ValueError('Without a file key it cannot request the '
+ 'stash information')
+ if not source_filename:
+ raise ValueError('Can request stash information only when '
+ 'using a file name.')
+ props = ['size']
+ if _verify_stash:
+ props += ['sha1']
+ stash_info = self.stash_info(_file_key, props)
+ if offset is True:
+ offset = stash_info['size']
+ elif offset is False:
+ if file_size != stash_info['size']:
+ raise ValueError(
+ 'For the file key "{0}" the server reported a size {1} '
+ 'while the file size is {2}'.format(
+ _file_key, stash_info['size'], file_size))
+ elif offset is not False and offset != stash_info['size']:
+ raise ValueError(
+ 'For the file key "{0}" the server reported a size {1} '
+ 'while the offset was {2}'.format(
+ _file_key, stash_info['size'], offset))
+
+ if _verify_stash:
+ # The SHA1 was also requested so calculate and compare it
+ assert 'sha1' in stash_info, \
+ 'sha1 not in stash info: {0}'.format(stash_info)
+ sha1 = hashlib.sha1()
+ bytes_to_read = offset
+ with open(source_filename, 'rb') as f:
+ while bytes_to_read > 0:
+ read_bytes = f.read(min(bytes_to_read, 1 << 20))
+ assert read_bytes # make sure we actually read bytes
+ bytes_to_read -= len(read_bytes)
+ sha1.update(read_bytes)
+ sha1 = sha1.hexdigest()
+ if sha1 != stash_info['sha1']:
+ raise ValueError(
+ 'The SHA1 of {0} bytes of the stashed "{1}" is {2} '
+ 'while the local file is {3}'.format(
+ offset, _file_key, stash_info['sha1'], sha1))
+
+ assert offset is not True
+ if _file_key and file_size is None:
+ assert offset is False
+
+ if _file_key and offset is False or offset == file_size:
pywikibot.log('Reused already upload file using '
'filekey "{0}"'.format(_file_key))
# TODO: Use sessionkey instead of filekey if necessary
@@ -5299,10 +5386,6 @@
# TODO: Dummy value to allow also Unicode names, see bug 73661
mime_filename = 'FAKE-NAME'
# upload local file
- # make sure file actually exists
- if not os.path.isfile(source_filename):
- raise ValueError("File '%s' does not exist."
- % source_filename)
throttle = True
filesize = os.path.getsize(source_filename)
chunked_upload = (chunk_size > 0 and chunk_size < filesize and
@@ -5313,7 +5396,6 @@
'action': 'upload', 'token': token, 'text': text,
'filename': file_page_title, 'comment': comment})
if chunked_upload:
- offset = _offset
if offset > 0:
pywikibot.log('Continuing upload from byte '
'{0}'.format(offset))
@@ -5343,13 +5425,16 @@
if error.code == u'uploaddisabled':
self._uploaddisabled = True
raise error
+ _file_key = data['filekey']
if 'warnings' in data and not ignore_all_warnings:
if callable(ignore_warnings):
+ if 'offset' not in data:
+ data['offset'] = True
if ignore_warnings(create_warnings_list(data)):
# Future warnings of this run can be ignored
ignore_warnings = True
ignore_all_warnings = True
- offset = result.get('offset', 0)
+ offset = data['offset']
continue
else:
return False
@@ -5357,7 +5442,6 @@
if 'offset' not in result:
result['offset'] = 0
break
- _file_key = data['filekey']
throttle = False
if 'offset' in data:
new_offset = int(data['offset'])
@@ -5376,13 +5460,16 @@
final_request['filekey'] = _file_key
break
else: # not chunked upload
- file_contents = f.read()
- filetype = (mimetypes.guess_type(source_filename)[0] or
- 'application/octet-stream')
- final_request.mime_params = {
- 'file': (file_contents, filetype.split('/'),
- {'filename': mime_filename})
- }
+ if _file_key:
+ final_request['filekey'] = _file_key
+ else:
+ file_contents = f.read()
+ filetype = (mimetypes.guess_type(source_filename)[0] or
+ 'application/octet-stream')
+ final_request.mime_params = {
+ 'file': (file_contents, filetype.split('/'),
+ {'filename': mime_filename})
+ }
else:
# upload by URL
if "upload_by_url" not in self.userinfo["rights"]:
@@ -5417,11 +5504,13 @@
_file_key = None
pywikibot.warning('No filekey defined.')
if not report_success:
+ if 'offset' not in result:
+ result['offset'] = True
if ignore_warnings(create_warnings_list(result)):
return self.upload(
filepage, source_filename, source_url, comment, text,
watch, True, chunk_size, _file_key,
- result.get('offset', False), report_success=False)
+ result['offset'], report_success=False)
else:
return False
warn('When ignore_warnings=False in APISite.upload will change '
diff --git a/tests/upload_tests.py b/tests/upload_tests.py
index 4d1b1ea..aed4a18 100644
--- a/tests/upload_tests.py
+++ b/tests/upload_tests.py
@@ -17,6 +17,8 @@
import pywikibot
+from pywikibot.data.api import APIError
+
from tests import _images_dir
from tests.aspects import unittest, TestCase
@@ -29,6 +31,9 @@
family = 'wikipedia'
code = 'test'
+
+ sounds_png = os.path.join(_images_dir, 'MP_sounds.png')
+ arrow_png = os.path.join(_images_dir, '1rightarrow.png')
def test_png(self):
"""Test uploading a png using Site.upload."""
@@ -46,6 +51,114 @@
comment='pywikibot test',
ignore_warnings=True, chunk_size=1024)
+ def _init_upload(self, chunk_size):
+ """Do an initial upload causing an abort because of warnings."""
+ def warn_callback(warnings):
+ """A simple callback not automatically finishing the upload."""
+ self.assertCountEqual([w.code for w in warnings], expected_warns)
+ # by now we know there are only two but just make sure
+ assert len(warnings) == len(expected_warns)
+ assert len(expected_warns) in [1, 2]
+ if len(expected_warns) == 2:
+ self.assertEqual(warnings[0].file_key, warnings[1].file_key)
+ self.assertEqual(warnings[0].offset, warnings[1].offset)
+ self._file_key = warnings[0].file_key
+ self._offset = warnings[0].offset
+
+ if chunk_size:
+ expected_warns = ['exists']
+ else:
+ expected_warns = ['duplicate', 'exists']
+
+ # First upload the warning with warnings enabled
+ page = pywikibot.FilePage(self.site, 'MP_sounds-pwb.png')
+ self.assertFalse(hasattr(self, '_file_key'))
+ self.site.upload(page, source_filename=self.sounds_png,
+ comment='pywikibot test', chunk_size=chunk_size,
+ ignore_warnings=warn_callback)
+
+ # Check that the warning happened and it's cached
+ self.assertTrue(hasattr(self, '_file_key'))
+ self.assertIs(self._offset, True)
+ self.assertRegex(self._file_key, r'[0-9a-z]+.[0-9a-z]+.\d+.png')
+ self._verify_stash()
+
+ def _verify_stash(self):
+ info = self.site.stash_info(self._file_key, ['size', 'sha1'])
+ if info['size'] == 1024:
+ self.assertEqual('3503db342c8dfb0a38db0682b7370ddd271fa163',
+ info['sha1'])
+ else:
+ self.assertEqual('0408a0f6a5e057e701f3aed96b0d1fb913c3d9d0',
+ info['sha1'])
+
+ def _finish_upload(self, chunk_size, file_name):
+ """Finish the upload."""
+ # Finish/continue upload with the given file key
+ page = pywikibot.FilePage(self.site, 'MP_sounds-pwb.png')
+ self.site.upload(page, source_filename=file_name,
+ comment='pywikibot test', chunk_size=chunk_size,
+ ignore_warnings=True, report_success=False,
+ _file_key=self._file_key, _offset=self._offset)
+
+ def _test_continue_filekey(self, chunk_size):
+ """Test uploading a chunk first and finish in a separate upload."""
+ self._init_upload(chunk_size)
+ self._finish_upload(chunk_size, self.sounds_png)
+
+ # Check if it's still cached
+ with self.assertRaises(APIError) as cm:
+ self.site.stash_info(self._file_key)
+ self.assertEqual(cm.exception.code, 'siiinvalidsessiondata')
+ self.assertTrue(cm.exception.info.startswith('File not found'),
+ 'info ({0}) did not start with '
+ '"File not found"'.format(cm.exception.info))
+
+ def test_continue_filekey_once(self):
+ """Test continuing to upload a file without using chunked mode."""
+ self._test_continue_filekey(0)
+
+ @unittest.expectedFailure # see T112416
+ def test_continue_filekey_chunked(self):
+ """Test continuing to upload a file with using chunked mode."""
+ self._test_continue_filekey(1024)
+
+ def test_sha1_missmatch(self):
+ """Test trying to continue with a different file."""
+ self._init_upload(1024)
+ with self.assertRaises(ValueError) as cm:
+ self._finish_upload(1024, self.arrow_png)
+ self.assertEqual(
+ str(cm.exception),
+ 'The SHA1 of 1024 bytes of the stashed "{0}" is '
+ '3503db342c8dfb0a38db0682b7370ddd271fa163 while the local file is '
+ '3dd334f11aa1e780d636416dc0649b96b67588b6'.format(self._file_key))
+ self._verify_stash()
+
+ def test_offset_missmatch(self):
+ """Test trying to continue with a different offset."""
+ self._init_upload(1024)
+ self._offset = 0
+ with self.assertRaises(ValueError) as cm:
+ self._finish_upload(1024, self.sounds_png)
+ self.assertEqual(
+ str(cm.exception),
+ 'For the file key "{0}" the server reported a size 1024 while the '
+ 'offset was 0'.format(self._file_key))
+ self._verify_stash()
+
+ def test_offset_oversize(self):
+ """Test trying to continue with an offset which is to large."""
+ self._init_upload(1024)
+ self._offset = 2000
+ with self.assertRaises(ValueError) as cm:
+ self._finish_upload(1024, self.sounds_png)
+ self.assertEqual(
+ str(cm.exception),
+ 'For the file key "{0}" the offset was set to 2000 while the file '
+ 'is only 1276 bytes large.'.format(self._file_key))
+ self._verify_stash()
+
if __name__ == '__main__':
try:
--
To view, visit https://gerrit.wikimedia.org/r/234851
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I610a0744d74dc5a2523d96eabde818f76dd6c5e8
Gerrit-PatchSet: 7
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>