jenkins-bot has submitted this change and it was merged.
Change subject: [FEAT] upload: Support -always option
......................................................................
[FEAT] upload: Support -always option
This adds an `-always` option to automatically upload files.
Bug: T106412
Change-Id: I478ff3152d8c086e2104cd2db21d01b12aead6ed
---
M scripts/upload.py
1 file changed, 78 insertions(+), 18 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/upload.py b/scripts/upload.py
index 5c50ab0..f70e228 100755
--- a/scripts/upload.py
+++ b/scripts/upload.py
@@ -22,6 +22,11 @@
'Ki': Kibibytes (1024 B)
'Mi': Mebibytes (1024x1024 B)
The suffixes are case insensitive.
+ -always Don't ask the user anything. This will imply -keep and
+ -noverify and require that either -abortonwarn or -ignorewarn
+ is defined for all. It will also require a valid file name and
+ description. It'll only overwrite files if -ignorewarn includes
+ the 'exists' warning.
It is possible to combine -abortonwarn and -ignorewarn so that if the specific
warning is given it won't apply the general one but more specific one. So if it
@@ -59,6 +64,7 @@
import pywikibot
import pywikibot.data.api
from pywikibot import config
+from pywikibot.bot import suggest_help, BaseBot
from pywikibot.tools import (
deprecated
)
@@ -72,14 +78,15 @@
from urllib import URLopener
-class UploadRobot:
+class UploadRobot(BaseBot):
"""Upload bot."""
def __init__(self, url, urlEncoding=None, description=u'',
useFilename=None, keepFilename=False,
verifyDescription=True, ignoreWarning=False,
- targetSite=None, uploadByUrl=False, aborts=[], chunk_size=0):
+ targetSite=None, uploadByUrl=False, aborts=[], chunk_size=0,
+ **kwargs):
"""
Constructor.
@@ -112,11 +119,23 @@
restartable) specified in bytes. If no value is specified the file
will be uploaded as whole.
@type chunk_size: integer
+ @param always: Disables any input, requires that either ignoreWarning or
+ aborts are set to True and that the description is also set. It will
+ overwrite verifyDescription to False and keepFilename to True.
+ @type always: bool
@deprecated: Using upload_image() is deprecated, use upload_file() with
file_url param instead
"""
+ super(UploadRobot, self).__init__(**kwargs)
+ always = self.getOption('always')
+ if (always and ignoreWarning is not True and aborts is not True):
+ raise ValueError('When always is set to True, either ignoreWarning '
+ 'or aborts must be set to True.')
+ if always and not description:
+ raise ValueError('When always is set to True the description must '
+ 'be set.')
self.url = url
if isinstance(self.url, basestring):
pywikibot.warning("url as string is deprecated. "
@@ -124,8 +143,8 @@
self.urlEncoding = urlEncoding
self.description = description
self.useFilename = useFilename
- self.keepFilename = keepFilename
- self.verifyDescription = verifyDescription
+ self.keepFilename = keepFilename or always
+ self.verifyDescription = verifyDescription and not always
self.ignoreWarning = ignoreWarning
self.aborts = aborts
self.chunk_size = chunk_size
@@ -249,6 +268,7 @@
pywikibot.warning("file_url is not given. "
"Set to self.url by default.")
+ always = self.getOption('always')
# Isolate the pure name
filename = file_url
# Filename may be either a URL or a local file path
@@ -262,6 +282,7 @@
pywikibot.output(
u"The filename on the target wiki will default to: %s"
% filename)
+ assert not always
newfn = pywikibot.input(
u'Enter a better name, or press enter to accept:')
if newfn != "":
@@ -277,8 +298,11 @@
first_check = True
while True:
if not first_check:
- filename = pywikibot.input('Enter a better name, or press '
- 'enter to skip the file:')
+ if always:
+ filename = None
+ else:
+ filename = pywikibot.input('Enter a better name, or press '
+ 'enter to skip the file:')
if not filename:
return None
first_check = False
@@ -291,22 +315,30 @@
'Invalid character(s): %s. Please try again' % c)
continue
if ext not in allowed_formats:
- if not pywikibot.input_yn(
+ if always:
+ pywikibot.output('File format is not one of '
+ '[{0}]'.format(' '.join(allowed_formats)))
+ continue
+ elif not pywikibot.input_yn(
u"File format is not one of [%s], but %s. Continue?"
% (u' '.join(allowed_formats), ext),
default=False, automatic_quit=False):
continue
potential_file_page = pywikibot.FilePage(self.targetSite, filename)
if potential_file_page.exists():
- if self.aborts is True:
+ overwrite = self._handle_warning('exists')
+ if overwrite is False:
pywikibot.output("File exists and you asked to abort. Skipping.")
return None
if potential_file_page.canBeEdited():
- if pywikibot.input_yn(u"File with name %s already exists. "
- "Would you like to change the name? "
- "(Otherwise file will be overwritten.)"
- % filename, default=True,
- automatic_quit=False):
+ if overwrite is None:
+ overwrite = not pywikibot.input_yn(
+ "File with name %s already exists. "
+ "Would you like to change the name? "
+ "(Otherwise file will be overwritten.)"
+ % filename, default=True,
+ automatic_quit=False)
+ if not overwrite:
continue
else:
break
@@ -341,6 +373,7 @@
u'\03{lightred}It is not possible to upload a file '
'without a summary/description.\03{default}')
+ assert not always
# if no description, default is 'yes'
if pywikibot.input_yn(
u'Do you want to change this description?',
@@ -468,6 +501,7 @@
url = u''
description = []
keepFilename = False
+ always = False
useFilename = None
verifyDescription = True
aborts = set()
@@ -480,7 +514,11 @@
# returns a list of non-global args, i.e. args for upload.py
for arg in pywikibot.handle_args(args):
if arg:
- if arg.startswith('-keep'):
+ if arg == '-always':
+ keepFilename = True
+ always = True
+ verifyDescription = False
+ elif arg.startswith('-keep'):
keepFilename = True
elif arg.startswith('-filename:'):
useFilename = arg[10:]
@@ -524,12 +562,35 @@
url = arg
else:
description.append(arg)
+ description = u' '.join(description)
while not ("://" in url or os.path.exists(url)):
if not url:
- pywikibot.output(u'No input filename given.')
+ error = 'No input filename given.'
else:
- pywikibot.output(u'Invalid input filename given. Try again.')
+ error = 'Invalid input filename given.'
+ if not always:
+ error += ' Try again.'
+ if always:
+ url = None
+ break
+ else:
+ pywikibot.output(error)
url = pywikibot.input(u'URL, file or directory where files are now:')
+ if always and ((aborts is not True and ignorewarn is not True) or
+ not description or url is None):
+ additional = ''
+ missing = []
+ if url is None:
+ missing += ['filename']
+ additional = error + ' '
+ if description is None:
+ missing += ['description']
+ if aborts is not True and ignorewarn is not True:
+ additional += ('Either -ignorewarn or -abortonwarn must be '
+ 'defined for all codes. ')
+ additional += 'Unable to run in -always mode'
+ suggest_help(missing_parameters=missing, additional_text=additional)
+ return False
if os.path.isdir(url):
file_list = []
for directory_info in os.walk(url):
@@ -538,12 +599,11 @@
url = file_list
else:
url = [url]
- description = u' '.join(description)
bot = UploadRobot(url, description=description, useFilename=useFilename,
keepFilename=keepFilename,
verifyDescription=verifyDescription,
aborts=aborts, ignoreWarning=ignorewarn,
- chunk_size=chunk_size)
+ chunk_size=chunk_size, always=always)
bot.run()
if __name__ == "__main__":
--
To view, visit https://gerrit.wikimedia.org/r/232708
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I478ff3152d8c086e2104cd2db21d01b12aead6ed
Gerrit-PatchSet: 5
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Ricordisamoa <ricordisamoa(a)openmailbox.org>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: [FEAT] upload: Support ignore warnings callback
......................................................................
[FEAT] upload: Support ignore warnings callback
This supports a callback method which is executed whenever an upload caused a
warning. This fixes the issue when multiple warnings are reported that the
upload method basically returned only one warning randomly.
It also provides a way to remove the success message as the method now returns
a boolean to indicate whether the upload succeeded.
Change-Id: I391bb2b7fd4a0f318c5b4537d4b615569413f447
---
M pywikibot/site.py
M scripts/upload.py
2 files changed, 141 insertions(+), 41 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/site.py b/pywikibot/site.py
index c09f399..3666f03 100644
--- a/pywikibot/site.py
+++ b/pywikibot/site.py
@@ -5180,7 +5180,7 @@
@deprecate_arg('imagepage', 'filepage')
def upload(self, filepage, source_filename=None, source_url=None,
comment=None, text=None, watch=False, ignore_warnings=False,
- chunk_size=0, _file_key=None, _offset=0):
+ chunk_size=0, _file_key=None, _offset=0, report_success=None):
"""Upload a file to the wiki.
Either source_filename or source_url, but not both, must be provided.
@@ -5195,8 +5195,16 @@
@param text: Initial page text; if this is not set, then
filepage.text will be used, or comment.
@param watch: If true, add filepage to the bot user's watchlist
- @param ignore_warnings: if true, ignore API warnings and force
- upload (for example, to overwrite an existing file); default False
+ @param ignore_warnings: It may be a static boolean, a callable returning
+ a boolean or an iterable. The callable gets a list of UploadWarning
+ instances and the iterable should contain the warning codes for
+ which an equivalent callable would return True if all UploadWarning
+ codes are in thet list. If the result is False it'll not continuing
+ uploading the file and otherwise disable any warning and
+ reattempting to upload the file. NOTE: If report_success is True or
+ None it'll raise an UploadWarning exception if the static boolean is
+ False.
+ @type ignore_warnings: bool or callable or iterable of str
@param chunk_size: The chunk size in bytesfor chunked uploading (see
U{https://www.mediawiki.org/wiki/API:Upload#Chunked_uploading}). It
will only upload in chunks, if the version number is 1.20 or higher
@@ -5209,7 +5217,23 @@
continue a previously canceled chunked upload. If False it treats
that as a finished upload. By default starts at 0.
@type _offset: int or bool
+ @param report_success: If the upload was successful it'll print a
+ success message and if ignore_warnings is set to False it'll
+ raise an UploadWarning if a warning occurred. If it's None (default)
+ it'll be True if ignore_warnings is a bool and False otherwise. If
+ it's True or None ignore_warnings must be a bool.
+ @return: It returns True if the upload was successful and False
+ otherwise.
+ @rtype: bool
"""
+ def create_warnings_list(response):
+ return [
+ api.UploadWarning(
+ warning,
+ upload_warnings.get(warning, '%(msg)s') % {'msg': data},
+ _file_key, response.get('offset', 0))
+ for warning, data in response['warnings'].items()]
+
upload_warnings = {
# map API warning codes to user error messages
# %(msg)s will be replaced by message string from API responsse
@@ -5240,6 +5264,22 @@
if not comment:
raise ValueError("APISite.upload: cannot upload file without "
"a summary/description.")
+ if report_success is None:
+ report_success = isinstance(ignore_warnings, bool)
+ if report_success is True:
+ if not isinstance(ignore_warnings, bool):
+ raise ValueError('report_success may only be set to True when '
+ 'ignore_warnings is a boolean')
+ issue_deprecation_warning('"ignore_warnings" as a boolean and '
+ '"report_success" is True or None',
+ '"report_success=False" or define '
+ '"ignore_warnings" as callable/iterable',
+ 2)
+ if isinstance(ignore_warnings, Iterable):
+ ignored_warnings = ignore_warnings
+ ignore_warnings = lambda warnings: all(w.code in ignored_warnings
+ for w in warnings)
+ ignore_all_warnings = not callable(ignore_warnings) and ignore_warnings
if text is None:
text = filepage.text
if not text:
@@ -5289,7 +5329,7 @@
'filesize': filesize,
'offset': offset,
'filename': file_page_title,
- 'ignorewarnings': ignore_warnings})
+ 'ignorewarnings': ignore_all_warnings})
req.mime_params['chunk'] = (chunk,
("application", "octet-stream"),
{'filename': mime_filename})
@@ -5303,7 +5343,16 @@
if error.code == u'uploaddisabled':
self._uploaddisabled = True
raise error
- if 'warnings' in data and not ignore_warnings:
+ if 'warnings' in data and not ignore_all_warnings:
+ if callable(ignore_warnings):
+ if ignore_warnings(create_warnings_list(data)):
+ # Future warnings of this run can be ignored
+ ignore_warnings = True
+ ignore_all_warnings = True
+ offset = result.get('offset', 0)
+ continue
+ else:
+ return False
result = data
if 'offset' not in result:
result['offset'] = 0
@@ -5345,7 +5394,7 @@
url=source_url, comment=comment, text=text, token=token)
if not result:
final_request['watch'] = watch
- final_request['ignorewarnings'] = ignore_warnings
+ final_request['ignorewarnings'] = ignore_all_warnings
try:
result = final_request.submit()
self._uploaddisabled = False
@@ -5357,10 +5406,7 @@
result = result["upload"]
pywikibot.debug(result, _logger)
- if "warnings" in result and not ignore_warnings:
- # TODO: Handle multiple warnings at the same time
- warning = list(result["warnings"].keys())[0]
- message = result["warnings"][warning]
+ if 'warnings' in result and not ignore_all_warnings:
if 'filekey' in result:
_file_key = result['filekey']
elif 'sessionkey' in result:
@@ -5370,6 +5416,24 @@
else:
_file_key = None
pywikibot.warning('No filekey defined.')
+ if not report_success:
+ if ignore_warnings(create_warnings_list(result)):
+ return self.upload(
+ filepage, source_filename, source_url, comment, text,
+ watch, True, chunk_size, _file_key,
+ result.get('offset', False), report_success=False)
+ else:
+ return False
+ warn('When ignore_warnings=False in APISite.upload will change '
+ 'from raising an UploadWarning into behaving like being a '
+ 'callable returning False.', DeprecationWarning, 2)
+ if len(result['warnings']) > 1:
+ warn('The upload returned {0} warnings: '
+ '{1}'.format(len(result['warnings']),
+ ', '.join(result['warnings'])),
+ UserWarning, 2)
+ warning = list(result["warnings"].keys())[0]
+ message = result["warnings"][warning]
raise pywikibot.UploadWarning(warning, upload_warnings[warning]
% {'msg': message},
file_key=_file_key,
@@ -5378,12 +5442,13 @@
elif "result" not in result:
pywikibot.output(u"Upload: unrecognized response: %s" % result)
if result["result"] == "Success":
- pywikibot.output(u"Upload successful.")
+ if report_success:
+ pywikibot.output(u"Upload successful.")
# If we receive a nochange, that would mean we're in simulation
# mode, don't attempt to access imageinfo
if "nochange" not in result:
filepage._load_file_revisions([result["imageinfo"]])
- return
+ return result['result'] == 'Success'
@deprecated_args(number='total',
repeat=None,
diff --git a/scripts/upload.py b/scripts/upload.py
index abf0ff7..5c50ab0 100755
--- a/scripts/upload.py
+++ b/scripts/upload.py
@@ -23,6 +23,13 @@
'Mi': Mebibytes (1024x1024 B)
The suffixes are case insensitive.
+It is possible to combine -abortonwarn and -ignorewarn so that if the specific
+warning is given it won't apply the general one but more specific one. So if it
+should ignore specific warnings and abort on the rest it's possible by defining
+no warning for -abortonwarn and the specific warnings for -ignorewarn. The order
+does not matter. If both are unspecific or a warning is specified by both, it'll
+prefer aborting.
+
If any other arguments are given, the first is either URL, filename or directory
to upload, and the rest is a proposed description to go with the upload. If none
of these are given, the user is asked for the directory, file or URL to upload.
@@ -197,6 +204,44 @@
t.close()
return tempname
+ def _handle_warning(self, warning):
+ """
+ Return whether the warning cause an abort or be ignored.
+
+ @param warning: The warning name
+ @type warning: str
+ @return: False if this warning should cause an abort, True if it should
+ be ignored or None if this warning has no default handler.
+ @rtype: bool or None
+ """
+ if self.aborts is not True:
+ if warning in self.aborts:
+ return False
+ if self.ignoreWarning is True or (self.ignoreWarning is not False and
+ warning in self.ignoreWarning):
+ return True
+ return None if self.aborts is not True else False
+
+ def _handle_warnings(self, warnings):
+ messages = '\n'.join('{0.code}: {0.info}'.format(warning)
+ for warning in sorted(warnings,
+ key=lambda w: w.code))
+ if len(warnings) > 1:
+ messages = '\n' + messages
+ pywikibot.output('We got the following warning(s): ' + messages)
+ answer = True
+ for warning in warnings:
+ this_answer = self._handle_warning(warning.code)
+ if this_answer is False:
+ answer = False
+ break
+ elif this_answer is None:
+ answer = None
+ if answer is None:
+ answer = pywikibot.input_yn(u"Do you want to ignore?",
+ default=False, automatic_quit=False)
+ return answer
+
def process_filename(self, file_url=None):
"""Return base filename portion of file_url."""
if not file_url:
@@ -351,54 +396,44 @@
pywikibot.output(u'Uploading file to %s via API...' % site)
+ success = False
try:
- apiIgnoreWarnings = False
if self.ignoreWarning is True:
apiIgnoreWarnings = True
+ else:
+ apiIgnoreWarnings = self._handle_warnings
if self.uploadByUrl:
- site.upload(imagepage, source_url=file_url,
- ignore_warnings=apiIgnoreWarnings,
- _file_key=_file_key, _offset=_offset)
+ success = site.upload(imagepage, source_url=file_url,
+ ignore_warnings=apiIgnoreWarnings,
+ _file_key=_file_key, _offset=_offset)
else:
if "://" in file_url:
temp = self.read_file_content(file_url)
else:
temp = file_url
- site.upload(imagepage, source_filename=temp,
- ignore_warnings=apiIgnoreWarnings,
- chunk_size=self.chunk_size,
- _file_key=_file_key, _offset=_offset)
+ success = site.upload(imagepage, source_filename=temp,
+ ignore_warnings=apiIgnoreWarnings,
+ chunk_size=self.chunk_size,
+ _file_key=_file_key, _offset=_offset)
- except pywikibot.data.api.UploadWarning as warn:
- pywikibot.output(
- u'We got a warning message: {0} - {1}'.format(warn.code, warn.message))
- if self.abort_on_warn(warn.code):
- answer = False
- elif self.ignore_on_warn(warn.code):
- answer = True
- else:
- answer = pywikibot.input_yn(u"Do you want to ignore?",
- default=False, automatic_quit=False)
- if answer:
- self.ignoreWarning = True
- self.keepFilename = True
- return self.upload_file(file_url, debug, warn.file_key, warn.offset)
- else:
- pywikibot.output(u"Upload aborted.")
- return
except pywikibot.data.api.APIError as error:
if error.code == u'uploaddisabled':
pywikibot.error("Upload error: Local file uploads are disabled on %s."
% site)
else:
pywikibot.error("Upload error: ", exc_info=True)
+ return None
except Exception:
pywikibot.error("Upload error: ", exc_info=True)
-
+ return None
else:
- # No warning, upload complete.
- pywikibot.output(u"Upload of %s successful." % filename)
- return filename # data['filename']
+ if success:
+ # No warning, upload complete.
+ pywikibot.output(u"Upload of %s successful." % filename)
+ return filename # data['filename']
+ else:
+ pywikibot.output(u"Upload aborted.")
+ return None
def run(self):
"""Run bot."""
--
To view, visit https://gerrit.wikimedia.org/r/233102
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I391bb2b7fd4a0f318c5b4537d4b615569413f447
Gerrit-PatchSet: 7
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>
Build Update for wikimedia/pywikibot-core
-------------------------------------
Build: #2698
Status: Passed
Duration: 43 minutes and 2 seconds
Commit: 2559642 (master)
Author: Fabian Neundorf
Message: [FIX] tests: Don't do hostname check if net=False
When using net=False it shouldn't do any hostname checks as the test is
intended to be run without any network activity. This only affects tests which
also have a site defined (e.g. using dry=True) as CheckHostnameMixin wouldn't
check if no hostname or sites are defined.
Also verifies that net is not False when only hostnames (and not sites) are
defined as hostnames is only there for hostname checks.
Change-Id: Ic60d6d24b489bf63284164411ad5cf1d4ba2ece6
View the changeset: https://github.com/wikimedia/pywikibot-core/compare/618a07d5895d...2559642c…
View the full build log and details: https://travis-ci.org/wikimedia/pywikibot-core/builds/76453237
--
You can configure recipients for build notifications in your .travis.yml file. See http://docs.travis-ci.com/user/notifications