jenkins-bot has submitted this change and it was merged.
Change subject: [FEAT] upload: Support ignore warnings callback ......................................................................
[FEAT] upload: Support ignore warnings callback
This supports a callback method which is executed whenever an upload caused a warning. This fixes the issue when multiple warnings are reported that the upload method basically returned only one warning randomly.
It also provides a way to remove the success message as the method now returns a boolean to indicate whether the upload succeeded.
Change-Id: I391bb2b7fd4a0f318c5b4537d4b615569413f447 --- M pywikibot/site.py M scripts/upload.py 2 files changed, 141 insertions(+), 41 deletions(-)
Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified
diff --git a/pywikibot/site.py b/pywikibot/site.py index c09f399..3666f03 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -5180,7 +5180,7 @@ @deprecate_arg('imagepage', 'filepage') def upload(self, filepage, source_filename=None, source_url=None, comment=None, text=None, watch=False, ignore_warnings=False, - chunk_size=0, _file_key=None, _offset=0): + chunk_size=0, _file_key=None, _offset=0, report_success=None): """Upload a file to the wiki.
Either source_filename or source_url, but not both, must be provided. @@ -5195,8 +5195,16 @@ @param text: Initial page text; if this is not set, then filepage.text will be used, or comment. @param watch: If true, add filepage to the bot user's watchlist - @param ignore_warnings: if true, ignore API warnings and force - upload (for example, to overwrite an existing file); default False + @param ignore_warnings: It may be a static boolean, a callable returning + a boolean or an iterable. The callable gets a list of UploadWarning + instances and the iterable should contain the warning codes for + which an equivalent callable would return True if all UploadWarning + codes are in thet list. If the result is False it'll not continuing + uploading the file and otherwise disable any warning and + reattempting to upload the file. NOTE: If report_success is True or + None it'll raise an UploadWarning exception if the static boolean is + False. + @type ignore_warnings: bool or callable or iterable of str @param chunk_size: The chunk size in bytesfor chunked uploading (see U{https://www.mediawiki.org/wiki/API:Upload#Chunked_uploading%7D). It will only upload in chunks, if the version number is 1.20 or higher @@ -5209,7 +5217,23 @@ continue a previously canceled chunked upload. If False it treats that as a finished upload. By default starts at 0. @type _offset: int or bool + @param report_success: If the upload was successful it'll print a + success message and if ignore_warnings is set to False it'll + raise an UploadWarning if a warning occurred. If it's None (default) + it'll be True if ignore_warnings is a bool and False otherwise. If + it's True or None ignore_warnings must be a bool. + @return: It returns True if the upload was successful and False + otherwise. + @rtype: bool """ + def create_warnings_list(response): + return [ + api.UploadWarning( + warning, + upload_warnings.get(warning, '%(msg)s') % {'msg': data}, + _file_key, response.get('offset', 0)) + for warning, data in response['warnings'].items()] + upload_warnings = { # map API warning codes to user error messages # %(msg)s will be replaced by message string from API responsse @@ -5240,6 +5264,22 @@ if not comment: raise ValueError("APISite.upload: cannot upload file without " "a summary/description.") + if report_success is None: + report_success = isinstance(ignore_warnings, bool) + if report_success is True: + if not isinstance(ignore_warnings, bool): + raise ValueError('report_success may only be set to True when ' + 'ignore_warnings is a boolean') + issue_deprecation_warning('"ignore_warnings" as a boolean and ' + '"report_success" is True or None', + '"report_success=False" or define ' + '"ignore_warnings" as callable/iterable', + 2) + if isinstance(ignore_warnings, Iterable): + ignored_warnings = ignore_warnings + ignore_warnings = lambda warnings: all(w.code in ignored_warnings + for w in warnings) + ignore_all_warnings = not callable(ignore_warnings) and ignore_warnings if text is None: text = filepage.text if not text: @@ -5289,7 +5329,7 @@ 'filesize': filesize, 'offset': offset, 'filename': file_page_title, - 'ignorewarnings': ignore_warnings}) + 'ignorewarnings': ignore_all_warnings}) req.mime_params['chunk'] = (chunk, ("application", "octet-stream"), {'filename': mime_filename}) @@ -5303,7 +5343,16 @@ if error.code == u'uploaddisabled': self._uploaddisabled = True raise error - if 'warnings' in data and not ignore_warnings: + if 'warnings' in data and not ignore_all_warnings: + if callable(ignore_warnings): + if ignore_warnings(create_warnings_list(data)): + # Future warnings of this run can be ignored + ignore_warnings = True + ignore_all_warnings = True + offset = result.get('offset', 0) + continue + else: + return False result = data if 'offset' not in result: result['offset'] = 0 @@ -5345,7 +5394,7 @@ url=source_url, comment=comment, text=text, token=token) if not result: final_request['watch'] = watch - final_request['ignorewarnings'] = ignore_warnings + final_request['ignorewarnings'] = ignore_all_warnings try: result = final_request.submit() self._uploaddisabled = False @@ -5357,10 +5406,7 @@ result = result["upload"] pywikibot.debug(result, _logger)
- if "warnings" in result and not ignore_warnings: - # TODO: Handle multiple warnings at the same time - warning = list(result["warnings"].keys())[0] - message = result["warnings"][warning] + if 'warnings' in result and not ignore_all_warnings: if 'filekey' in result: _file_key = result['filekey'] elif 'sessionkey' in result: @@ -5370,6 +5416,24 @@ else: _file_key = None pywikibot.warning('No filekey defined.') + if not report_success: + if ignore_warnings(create_warnings_list(result)): + return self.upload( + filepage, source_filename, source_url, comment, text, + watch, True, chunk_size, _file_key, + result.get('offset', False), report_success=False) + else: + return False + warn('When ignore_warnings=False in APISite.upload will change ' + 'from raising an UploadWarning into behaving like being a ' + 'callable returning False.', DeprecationWarning, 2) + if len(result['warnings']) > 1: + warn('The upload returned {0} warnings: ' + '{1}'.format(len(result['warnings']), + ', '.join(result['warnings'])), + UserWarning, 2) + warning = list(result["warnings"].keys())[0] + message = result["warnings"][warning] raise pywikibot.UploadWarning(warning, upload_warnings[warning] % {'msg': message}, file_key=_file_key, @@ -5378,12 +5442,13 @@ elif "result" not in result: pywikibot.output(u"Upload: unrecognized response: %s" % result) if result["result"] == "Success": - pywikibot.output(u"Upload successful.") + if report_success: + pywikibot.output(u"Upload successful.") # If we receive a nochange, that would mean we're in simulation # mode, don't attempt to access imageinfo if "nochange" not in result: filepage._load_file_revisions([result["imageinfo"]]) - return + return result['result'] == 'Success'
@deprecated_args(number='total', repeat=None, diff --git a/scripts/upload.py b/scripts/upload.py index abf0ff7..5c50ab0 100755 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -23,6 +23,13 @@ 'Mi': Mebibytes (1024x1024 B) The suffixes are case insensitive.
+It is possible to combine -abortonwarn and -ignorewarn so that if the specific +warning is given it won't apply the general one but more specific one. So if it +should ignore specific warnings and abort on the rest it's possible by defining +no warning for -abortonwarn and the specific warnings for -ignorewarn. The order +does not matter. If both are unspecific or a warning is specified by both, it'll +prefer aborting. + If any other arguments are given, the first is either URL, filename or directory to upload, and the rest is a proposed description to go with the upload. If none of these are given, the user is asked for the directory, file or URL to upload. @@ -197,6 +204,44 @@ t.close() return tempname
+ def _handle_warning(self, warning): + """ + Return whether the warning cause an abort or be ignored. + + @param warning: The warning name + @type warning: str + @return: False if this warning should cause an abort, True if it should + be ignored or None if this warning has no default handler. + @rtype: bool or None + """ + if self.aborts is not True: + if warning in self.aborts: + return False + if self.ignoreWarning is True or (self.ignoreWarning is not False and + warning in self.ignoreWarning): + return True + return None if self.aborts is not True else False + + def _handle_warnings(self, warnings): + messages = '\n'.join('{0.code}: {0.info}'.format(warning) + for warning in sorted(warnings, + key=lambda w: w.code)) + if len(warnings) > 1: + messages = '\n' + messages + pywikibot.output('We got the following warning(s): ' + messages) + answer = True + for warning in warnings: + this_answer = self._handle_warning(warning.code) + if this_answer is False: + answer = False + break + elif this_answer is None: + answer = None + if answer is None: + answer = pywikibot.input_yn(u"Do you want to ignore?", + default=False, automatic_quit=False) + return answer + def process_filename(self, file_url=None): """Return base filename portion of file_url.""" if not file_url: @@ -351,54 +396,44 @@
pywikibot.output(u'Uploading file to %s via API...' % site)
+ success = False try: - apiIgnoreWarnings = False if self.ignoreWarning is True: apiIgnoreWarnings = True + else: + apiIgnoreWarnings = self._handle_warnings if self.uploadByUrl: - site.upload(imagepage, source_url=file_url, - ignore_warnings=apiIgnoreWarnings, - _file_key=_file_key, _offset=_offset) + success = site.upload(imagepage, source_url=file_url, + ignore_warnings=apiIgnoreWarnings, + _file_key=_file_key, _offset=_offset) else: if "://" in file_url: temp = self.read_file_content(file_url) else: temp = file_url - site.upload(imagepage, source_filename=temp, - ignore_warnings=apiIgnoreWarnings, - chunk_size=self.chunk_size, - _file_key=_file_key, _offset=_offset) + success = site.upload(imagepage, source_filename=temp, + ignore_warnings=apiIgnoreWarnings, + chunk_size=self.chunk_size, + _file_key=_file_key, _offset=_offset)
- except pywikibot.data.api.UploadWarning as warn: - pywikibot.output( - u'We got a warning message: {0} - {1}'.format(warn.code, warn.message)) - if self.abort_on_warn(warn.code): - answer = False - elif self.ignore_on_warn(warn.code): - answer = True - else: - answer = pywikibot.input_yn(u"Do you want to ignore?", - default=False, automatic_quit=False) - if answer: - self.ignoreWarning = True - self.keepFilename = True - return self.upload_file(file_url, debug, warn.file_key, warn.offset) - else: - pywikibot.output(u"Upload aborted.") - return except pywikibot.data.api.APIError as error: if error.code == u'uploaddisabled': pywikibot.error("Upload error: Local file uploads are disabled on %s." % site) else: pywikibot.error("Upload error: ", exc_info=True) + return None except Exception: pywikibot.error("Upload error: ", exc_info=True) - + return None else: - # No warning, upload complete. - pywikibot.output(u"Upload of %s successful." % filename) - return filename # data['filename'] + if success: + # No warning, upload complete. + pywikibot.output(u"Upload of %s successful." % filename) + return filename # data['filename'] + else: + pywikibot.output(u"Upload aborted.") + return None
def run(self): """Run bot."""