jenkins-bot submitted this change.
[IMPR] Provide an Uploader class to upload files
currently the upload function has an overhead of parameter which are
called recursivly. Provide an Uploader class to hold the parameters.
- remove UploadMixin and add upload method to APISite which just
creates the Uploader and calls the upload() method
- add Uploader class which ruses the upload parameters to hold them
- upload() method to call the inner _upload method
- documentation added
Change-Id: Ifc83112400cbe2ba3a2ebf89099166c0c0fc3398
---
M docs/api_ref/pywikibot.site.rst
M pywikibot/site/_apisite.py
M pywikibot/site/_upload.py
3 files changed, 209 insertions(+), 179 deletions(-)
diff --git a/docs/api_ref/pywikibot.site.rst b/docs/api_ref/pywikibot.site.rst
index af145a8..cd97c76 100644
--- a/docs/api_ref/pywikibot.site.rst
+++ b/docs/api_ref/pywikibot.site.rst
@@ -35,3 +35,8 @@
TokenWallet Package
-------------------
.. automodule:: pywikibot.site._tokenwallet
+
+Uploader Package
+----------------
+.. automodule:: pywikibot.site._upload
+
diff --git a/pywikibot/site/_apisite.py b/pywikibot/site/_apisite.py
index 07fd907..a1bbd11 100644
--- a/pywikibot/site/_apisite.py
+++ b/pywikibot/site/_apisite.py
@@ -65,7 +65,7 @@
from pywikibot.site._namespace import Namespace
from pywikibot.site._siteinfo import Siteinfo
from pywikibot.site._tokenwallet import TokenWallet
-from pywikibot.site._upload import UploadMixin
+from pywikibot.site._upload import Uploader
from pywikibot.tools import (
MediaWikiVersion,
deprecated,
@@ -93,7 +93,6 @@
ThanksMixin,
UrlShortenerMixin,
WikibaseClientMixin,
- UploadMixin,
):
"""API interface to MediaWiki site.
@@ -2521,6 +2520,29 @@
siiprop=props)
return req.submit()['query']['stashimageinfo'][0]
+ @need_right('upload')
+ def upload(self, filepage, **kwargs) -> bool:
+ """Upload a file to the wiki.
+
+ :see: https://www.mediawiki.org/wiki/API:Upload
+
+ Either source_filename or source_url, but not both, must be provided.
+
+ .. versionchanged:: 6.0
+ keyword arguments required for all parameters except `filepage`
+
+ .. versionchanged:: 6.2:
+ asynchronous upload is used if `asynchronous` parameter is set.
+
+ For keyword arguments refer :class:`pywikibot.site._upload.Uploader`
+
+ :param filepage: a FilePage object from which the wiki-name of the
+ file will be obtained.
+ :return: It returns True if the upload was successful and False
+ otherwise.
+ """
+ return Uploader(self, filepage, **kwargs).upload()
+
def get_property_names(self, force: bool = False):
"""
Get property names for pages_with_property().
diff --git a/pywikibot/site/_upload.py b/pywikibot/site/_upload.py
index d09a505..a8608ae 100644
--- a/pywikibot/site/_upload.py
+++ b/pywikibot/site/_upload.py
@@ -7,7 +7,7 @@
import mimetypes
import os
from collections.abc import Iterable
-from typing import Optional, Union
+from typing import Optional
from warnings import warn
import pywikibot
@@ -16,95 +16,122 @@
Error,
UploadError,
)
-from pywikibot.site._decorators import need_right
from pywikibot.tools import compute_file_hash
-__all__ = ('UploadMixin', )
+__all__ = ('Uploader', )
_logger = 'wiki.apisite'
-class UploadMixin:
+class Uploader:
- """Site mixin to Upload a file to the wiki.
+ """Uploader class to upload a file to the wiki.
.. versionadded:: 7.1
+
+ :param site: The current site to work on
+ :param filepage: a FilePage object from which the wiki-name of the
+ file will be obtained.
+ :param source_filename: path to the file to be uploaded
+ :param source_url: URL of the file to be uploaded
+ :param comment: Edit summary; if this is not provided, then
+ filepage.text will be used. An empty summary is not permitted.
+ This may also serve as the initial page text (see below).
+ :param text: Initial page text; if this is not set, then
+ filepage.text will be used, or comment.
+ :param watch: If true, add filepage to the bot user's watchlist
+ :param chunk_size: The chunk size in bytes for chunked uploading (see
+ https://www.mediawiki.org/wiki/API:Upload#Chunked_uploading).
+ It will only upload in chunks, if the chunk size is positive
+ but lower than the file size.
+ :param asynchronous: Make potentially large file operations
+ asynchronous on the server side when possible.
+ :param ignore_warnings: It may be a static boolean, a callable
+ returning a boolean or an iterable. The callable gets a list of
+ UploadError instances and the iterable should contain the warning
+ codes for which an equivalent callable would return True if all
+ UploadError codes are in thet list. If the result is False it'll
+ not continue uploading the file and otherwise disable any warning
+ and reattempt to upload the file. NOTE: If report_success is True
+ or None it'll raise an UploadError exception if the static
+ boolean is False.
+ :type ignore_warnings: bool or callable or iterable of str
+ :param report_success: If the upload was successful it'll print a
+ success message and if ignore_warnings is set to False it'll
+ raise an UploadError if a warning occurred. If it's None
+ (default) it'll be True if ignore_warnings is a bool and False
+ otherwise. If it's True or None ignore_warnings must be a bool.
"""
- @need_right('upload')
- def upload(self, filepage, *,
- source_filename: Optional[str] = None,
- source_url: Optional[str] = None,
- comment: Optional[str] = None,
- text: Optional[str] = None,
- watch: bool = False,
- ignore_warnings=False,
- chunk_size: int = 0,
- asynchronous: bool = False,
- report_success: Optional[bool] = None,
- _file_key: Optional[str] = None,
- _offset: Union[bool, int] = 0) -> bool:
+ def __init__(self,
+ site: 'pywikibot.site.APISite',
+ filepage: 'pywikibot.FilePage',
+ *,
+ source_filename: Optional[str] = None,
+ source_url: Optional[str] = None,
+ comment: Optional[str] = None,
+ text: Optional[str] = None,
+ watch: bool = False,
+ chunk_size: int = 0,
+ asynchronous: bool = False,
+ ignore_warnings=False,
+ report_success: Optional[bool] = None) -> None:
+ """Initializer."""
+ self.site = site
+ self.filepage = filepage
+ self.comment = comment
+ self.text = text
+ self.watch = watch
+ self.ignore_warnings = ignore_warnings
+ self.chunk_size = chunk_size
+ self.asynchronous = asynchronous
+ self.report_success = report_success
+
+ if source_filename and source_url:
+ raise ValueError('APISite.upload: must provide either '
+ 'source_filename or source_url, not both.')
+
+ self.filename = source_filename
+ self.url = source_url
+
+ def upload(self) -> bool:
+ """Check for required parameters to upload and run the job.
+
+ :return: Whether the upload was successful.
"""
- Upload a file to the wiki.
+ if self.comment is None:
+ self.comment = self.filepage.text
+ if not self.comment:
+ raise ValueError('APISite.upload: cannot upload file without '
+ 'a summary/description.')
- :see: https://www.mediawiki.org/wiki/API:Upload
+ if self.text is None:
+ self.text = self.filepage.text
+ if not self.text:
+ self.text = self.comment
- Either source_filename or source_url, but not both, must be provided.
+ return self._upload(self.ignore_warnings, self.report_success)
- .. versionchanged:: 6.0
- keyword arguments required for all parameters except *filepage*
+ def _upload(self, ignore_warnings, report_success,
+ file_key=None, offset=0) -> bool:
+ """Recursive Upload method.
- .. versionchanged:: 6.2:
- asynchronous upload is used if *asynchronous* parameter is set.
-
- :param filepage: a FilePage object from which the wiki-name of the
- file will be obtained.
- :param source_filename: path to the file to be uploaded
- :param source_url: URL of the file to be uploaded
- :param comment: Edit summary; if this is not provided, then
- filepage.text will be used. An empty summary is not permitted.
- This may also serve as the initial page text (see below).
- :param text: Initial page text; if this is not set, then
- filepage.text will be used, or comment.
- :param watch: If true, add filepage to the bot user's watchlist
- :param ignore_warnings: It may be a static boolean, a callable
- returning a boolean or an iterable. The callable gets a list of
- UploadError instances and the iterable should contain the warning
- codes for which an equivalent callable would return True if all
- UploadError codes are in thet list. If the result is False it'll
- not continue uploading the file and otherwise disable any warning
- and reattempt to upload the file. NOTE: If report_success is True
- or None it'll raise an UploadError exception if the static
- boolean is False.
- :type ignore_warnings: bool or callable or iterable of str
- :param chunk_size: The chunk size in bytes for chunked uploading (see
- https://www.mediawiki.org/wiki/API:Upload#Chunked_uploading).
- It will only upload in chunks, if the chunk size is positive
- but lower than the file size.
- :param asynchronous: Make potentially large file operations
- asynchronous on the server side when possible.
- :param report_success: If the upload was successful it'll print a
- success message and if ignore_warnings is set to False it'll
- raise an UploadError if a warning occurred. If it's None
- (default) it'll be True if ignore_warnings is a bool and False
- otherwise. If it's True or None ignore_warnings must be a bool.
- :param _file_key: Private parameter for upload recurion. Reuses
- an already uploaded file using the filekey. If None (default)
- it will upload the file.
- :param _offset: Private parameter for upload recurion. When
- file_key is not None this can be an integer to continue a
- previously canceled chunked upload. If False it treats that
- as a finished upload. If True it requests the stash info from
- the server to determine the offset. By default starts at 0.
- :return: It returns True if the upload was successful and False
- otherwise.
+ :param file_key: Reuses an already uploaded file using the
+ filekey. If None (default) it will upload the file.
+ :param offset: When file_key is not None this can be an integer
+ to continue a previously canceled chunked upload. If False
+ it treats that as a finished upload. If True it requests the
+ stash info from the server to determine the offset. By
+ default starts at 0.
+ :return: Whether the upload was successful.
"""
+
def create_warnings_list(response):
return [
UploadError(
warning,
upload_warnings.get(warning, '{msg}').format(msg=data),
- _file_key, response['offset'])
+ file_key, response['offset'])
for warning, data in response['warnings'].items()]
# some warning keys have been changed
@@ -139,16 +166,8 @@
}
# An offset != 0 doesn't make sense without a file key
- assert(_offset == 0 or _file_key is not None)
- # check for required parameters
- if source_filename and source_url:
- raise ValueError('APISite.upload: must provide either '
- 'source_filename or source_url, not both.')
- if comment is None:
- comment = filepage.text
- if not comment:
- raise ValueError('APISite.upload: cannot upload file without '
- 'a summary/description.')
+ assert(offset == 0 or file_key is not None)
+
if report_success is None:
report_success = isinstance(ignore_warnings, bool)
if report_success is True:
@@ -162,51 +181,47 @@
return all(w.code in ignored_warnings for w in warnings)
ignore_all_warnings = not callable(ignore_warnings) and ignore_warnings
- if text is None:
- text = filepage.text
- if not text:
- text = comment
- token = self.tokens['edit']
+
+ token = self.site.tokens['edit']
result = None
- file_page_title = filepage.title(with_ns=False)
+ file_page_title = self.filepage.title(with_ns=False)
file_size = None
- offset = _offset
# make sure file actually exists
- if source_filename:
- if os.path.isfile(source_filename):
- file_size = os.path.getsize(source_filename)
+ if self.filename:
+ if os.path.isfile(self.filename):
+ file_size = os.path.getsize(self.filename)
elif offset is not False:
raise ValueError("File '{}' does not exist."
- .format(source_filename))
+ .format(self.filename))
# Verify the stash when a file key and offset is given:
# requests the SHA1 and file size uploaded and compares it to
- # the local file. Also verify that _offset is matching the
- # file size if the _offset is an int. If _offset is False if
+ # the local file. Also verify that offset is matching the
+ # file size if the offset is an int. If offset is False if
# verifies that the file size match with the local file.
verify_stash = False
- if source_filename and _file_key:
+ if self.filename and file_key:
assert offset is False or file_size is not None
verify_stash = True
if (offset is not False and offset is not True
and offset > file_size):
raise ValueError(
'For the file key "{}" the offset was set to {} '
- 'while the file is only {} bytes large.'.format(
- _file_key, offset, file_size))
+ 'while the file is only {} bytes large.'
+ .format(file_key, offset, file_size))
if verify_stash or offset is True:
- if not _file_key:
+ if not file_key:
raise ValueError('Without a file key it cannot request the '
'stash information')
- if not source_filename:
+ if not self.filename:
raise ValueError('Can request stash information only when '
'using a file name.')
props = ['size']
if verify_stash:
props += ['sha1']
- stash_info = self.stash_info(_file_key, props)
+ stash_info = self.site.stash_info(file_key, props)
if offset is True:
offset = stash_info['size']
elif offset is False:
@@ -214,55 +229,55 @@
raise ValueError(
'For the file key "{}" the server reported a size '
'{} while the file size is {}'
- .format(_file_key, stash_info['size'], file_size))
+ .format(file_key, stash_info['size'], file_size))
elif offset is not False and offset != stash_info['size']:
raise ValueError(
'For the file key "{}" the server reported a size {} '
- 'while the offset was {}'.format(
- _file_key, stash_info['size'], offset))
+ 'while the offset was {}'
+ .format(file_key, stash_info['size'], offset))
if verify_stash:
# The SHA1 was also requested so calculate and compare it
assert 'sha1' in stash_info, \
'sha1 not in stash info: {}'.format(stash_info)
- sha1 = compute_file_hash(source_filename, bytes_to_read=offset)
+ sha1 = compute_file_hash(self.filename, bytes_to_read=offset)
if sha1 != stash_info['sha1']:
raise ValueError(
'The SHA1 of {} bytes of the stashed "{}" is {} '
- 'while the local file is {}'.format(
- offset, _file_key, stash_info['sha1'], sha1))
+ 'while the local file is {}'
+ .format(offset, file_key, stash_info['sha1'], sha1))
assert offset is not True
- if _file_key and file_size is None:
+ if file_key and file_size is None:
assert offset is False
- if _file_key and offset is False or offset == file_size:
- pywikibot.log('Reused already upload file using '
- 'filekey "{}"'.format(_file_key))
+ if file_key and offset is False or offset == file_size:
+ pywikibot.log('Reused already upload file using filekey "{}"'
+ .format(file_key))
# TODO: Use sessionkey instead of filekey if necessary
- final_request = self._request(
+ final_request = self.site._request(
parameters={
'action': 'upload',
'token': token,
'filename': file_page_title,
- 'comment': comment,
- 'text': text,
- 'async': asynchronous,
- 'filekey': _file_key
+ 'comment': self.comment,
+ 'text': self.text,
+ 'async': self.asynchronous,
+ 'filekey': file_key
})
- elif source_filename:
+ elif self.filename:
# TODO: Dummy value to allow also Unicode names, see bug T75661
mime_filename = 'FAKE-NAME'
# upload local file
throttle = True
- filesize = os.path.getsize(source_filename)
- chunked_upload = 0 < chunk_size < filesize
- with open(source_filename, 'rb') as f:
- final_request = self._request(
+ filesize = os.path.getsize(self.filename)
+ chunked_upload = 0 < self.chunk_size < filesize
+ with open(self.filename, 'rb') as f:
+ final_request = self.site._request(
throttle=throttle, parameters={
- 'action': 'upload', 'token': token, 'text': text,
- 'filename': file_page_title, 'comment': comment})
+ 'action': 'upload', 'token': token, 'text': self.text,
+ 'filename': file_page_title, 'comment': self.comment})
if chunked_upload:
if offset > 0:
pywikibot.log('Continuing upload from byte {}'
@@ -272,20 +287,20 @@
if poll:
# run a poll; not possible in first iteration
- assert _file_key
- req = self._simple_request(
+ assert file_key
+ req = self.site._simple_request(
action='upload',
token=token,
- filekey=_file_key,
+ filekey=file_key,
checkstatus=True)
else:
f.seek(offset)
- chunk = f.read(chunk_size)
+ chunk = f.read(self.chunk_size)
# workaround (hack) for T132676
# append another '\r' so that one is the payload
# and the second is used for newline when mangled
# by email package.
- if (len(chunk) < chunk_size
+ if (len(chunk) < self.chunk_size
or (offset + len(chunk)) == filesize
and chunk[-1] == b'\r'[0]):
chunk += b'\r'
@@ -295,7 +310,7 @@
('application', 'octet-stream'),
{'filename': mime_filename})
}
- req = self._request(
+ req = self.site._request(
throttle=throttle,
mime=mime_params,
parameters={
@@ -305,19 +320,19 @@
'filesize': filesize,
'offset': offset,
'filename': file_page_title,
- 'async': asynchronous,
+ 'async': self.asynchronous,
'ignorewarnings': ignore_all_warnings})
- if _file_key:
- req['filekey'] = _file_key
+ if file_key:
+ req['filekey'] = file_key
try:
data = req.submit()['upload']
- self._uploaddisabled = False
+ self.site._uploaddisabled = False
except APIError as error:
# TODO: catch and process foreseeable errors
if error.code == 'uploaddisabled':
- self._uploaddisabled = True
+ self.site._uploaddisabled = True
elif error.code == 'stashfailed' \
and 'offset' in error.other:
# TODO: Ask MediaWiki to change this
@@ -348,7 +363,7 @@
break
# Polls may not contain file key in response
- _file_key = data.get('filekey', _file_key)
+ file_key = data.get('filekey', file_key)
if data['result'] == 'Warning':
assert('warnings' in data
and not ignore_all_warnings)
@@ -366,17 +381,8 @@
# Future warnings of this run
# can be ignored
if restart:
- return self.upload(
- filepage,
- source_filename=source_filename,
- source_url=source_url,
- comment=comment,
- text=text,
- watch=watch,
+ return self._upload(
ignore_warnings=True,
- chunk_size=chunk_size,
- _file_key=None,
- _offset=0,
report_success=False
)
@@ -395,10 +401,10 @@
new_offset = int(data['offset'])
if offset + len(chunk) != new_offset:
pywikibot.log('Old offset: {}; Returned '
- 'offset: {}; Chunk size: '
- '{}'.format(offset,
- new_offset,
- len(chunk)))
+ 'offset: {}; Chunk size: {}'
+ .format(offset,
+ new_offset,
+ len(chunk)))
pywikibot.warning('Unexpected offset.')
offset = new_offset
else:
@@ -410,19 +416,19 @@
'assemble chunks.')
elif data['result'] == 'Success': # finished
pywikibot.log('Finished uploading last chunk.')
- final_request['filekey'] = _file_key
- final_request['async'] = asynchronous
+ final_request['filekey'] = file_key
+ final_request['async'] = self.asynchronous
break
else:
raise Error(
'Unrecognized result: %s' % data['result'])
else: # not chunked upload
- if _file_key:
- final_request['filekey'] = _file_key
+ if file_key:
+ final_request['filekey'] = file_key
else:
file_contents = f.read()
- filetype = (mimetypes.guess_type(source_filename)[0]
+ filetype = (mimetypes.guess_type(self.filename)[0]
or 'application/octet-stream')
final_request.mime = {
'file': (file_contents, filetype.split('/'),
@@ -430,25 +436,25 @@
}
else:
# upload by URL
- if not self.has_right('upload_by_url'):
+ if not self.site.has_right('upload_by_url'):
raise Error(
"User '{}' is not authorized to upload by URL on site {}."
- .format(self.user(), self))
- final_request = self._simple_request(
- action='upload', filename=file_page_title,
- url=source_url, comment=comment, text=text, token=token)
+ .format(self.site.user(), self))
+ final_request = self.site._simple_request(
+ action='upload', filename=file_page_title, url=self.url,
+ comment=self.comment, text=self.text, token=token)
while True:
if not result:
- final_request['watch'] = watch
+ final_request['watch'] = self.watch
final_request['ignorewarnings'] = ignore_all_warnings
try:
result = final_request.submit()
- self._uploaddisabled = False
+ self.site._uploaddisabled = False
except APIError as error:
# TODO: catch and process foreseeable errors
if error.code == 'uploaddisabled':
- self._uploaddisabled = True
+ self.site._uploaddisabled = True
raise error
result = result['upload']
pywikibot.debug(result, _logger)
@@ -459,32 +465,29 @@
if result['result'] == 'Warning':
assert 'warnings' in result and not ignore_all_warnings
- if source_filename:
+ if self.filename:
if 'filekey' in result:
- _file_key = result['filekey']
+ file_key = result['filekey']
elif 'sessionkey' in result:
# TODO: Probably needs to be reflected in the API call
# above
- _file_key = result['sessionkey']
+ file_key = result['sessionkey']
pywikibot.warning(
'Using sessionkey instead of filekey.')
else:
- _file_key = None
+ file_key = None
pywikibot.warning('No filekey defined.')
else:
- _file_key = None
+ file_key = None
if not report_success:
- result.setdefault('offset', bool(source_filename))
- offset = result['offset'] if source_filename else False
+ result.setdefault('offset', bool(self.filename))
+ offset = result['offset'] if self.filename else False
if ignore_warnings(create_warnings_list(result)):
- return self.upload(
- filepage, source_filename=source_filename,
- source_url=source_url, comment=comment,
- text=text, watch=watch, ignore_warnings=True,
- chunk_size=chunk_size, asynchronous=asynchronous,
- _file_key=_file_key, _offset=offset,
- report_success=False)
+ return self._upload(ignore_warnings=True,
+ report_success=False,
+ file_key=file_key,
+ offset=offset)
return False
if len(result['warnings']) > 1:
@@ -498,18 +501,18 @@
raise UploadError(warning,
upload_warnings[warning]
.format(msg=message),
- file_key=_file_key,
+ file_key=file_key,
offset=result.get('offset', False))
if result['result'] == 'Poll':
# Polling is meaningless without a file key
- assert _file_key
+ assert file_key
pywikibot.log('Waiting for upload to be published.')
result = None
- final_request = self._simple_request(
+ final_request = self.site._simple_request(
action='upload',
token=token,
- filekey=_file_key,
+ filekey=file_key,
checkstatus=True)
continue
@@ -519,7 +522,7 @@
# If we receive a nochange, that would mean we're in simulation
# mode, don't attempt to access imageinfo
if 'nochange' not in result:
- filepage._load_file_revisions([result['imageinfo']])
+ self.filepage._load_file_revisions([result['imageinfo']])
return True
raise Error('Unrecognized result: %s' % data['result'])
To view, visit change 769276. To unsubscribe, or for help writing mail filters, visit settings.