jenkins-bot has submitted this change and it was merged.

Change subject: [IMPR] Separate scripts and library part ......................................................................

[IMPR] Separate scripts and library part

- pywikibot framework is designed to separate scripts from library parts Move UploadRobot to pywikibot.specialbots. Other bots may follow. - By importing UploadRobot in upload.py other scripts won't break when they import upload and use upload.UploadRobot - Anyway fix import in all known scripts using UploadRobot

Bug: T89499 Change-Id: I3a1ec2089930d1a0a9008008c34bc151b27fe5fe --- A pywikibot/specialbots.py M scripts/data_ingestion.py M scripts/flickrripper.py M scripts/imagecopy.py M scripts/imagecopy_self.py M scripts/imageharvest.py M scripts/imagetransfer.py M scripts/panoramiopicker.py M scripts/upload.py 9 files changed, 501 insertions(+), 484 deletions(-)

Approvals: Lokal Profil: Looks good to me, approved jenkins-bot: Verified

diff --git a/pywikibot/specialbots.py b/pywikibot/specialbots.py new file mode 100644 index 0000000..c524269 --- /dev/null +++ b/pywikibot/specialbots.py @@ -0,0 +1,452 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +"""Library containing special bots.""" +# +# (C) Rob W.W. Hooft, Andre Engels 2003-2004 +# (C) Pywikibot team, 2003-2016 +# +# Distributed under the terms of the MIT license. +# +from __future__ import absolute_import, unicode_literals + +__version__ = '$Id$' +# + +import os +import sys +import tempfile +import time + +import pywikibot +import pywikibot.data.api + +from pywikibot import config + +from pywikibot.bot import BaseBot +from pywikibot.tools import ( + deprecated +) +from pywikibot.tools.formatter import color_format + +if sys.version_info[0] > 2: + from urllib.parse import urlparse + from urllib.request import URLopener + + basestring = (str,) +else: + from urllib import URLopener + from urlparse import urlparse + + +class UploadRobot(BaseBot): + + """Upload bot.""" + + def __init__(self, url, urlEncoding=None, description=u'', + useFilename=None, keepFilename=False, + verifyDescription=True, ignoreWarning=False, + targetSite=None, uploadByUrl=False, aborts=[], chunk_size=0, + **kwargs): + """ + Constructor. + + @param url: path to url or local file (deprecated), or list of urls or + paths to local files. + @type url: string (deprecated) or list + @param description: Description of file for its page. If multiple files + are uploading the same description is used for every file. + @type description: string + @param useFilename: Specify title of the file's page. If multiple + files are uploading it asks to change the name for second, third, + etc. files, otherwise the last file will overwrite the other. + @type useFilename: string + @param keepFilename: Set to True to keep original names of urls and + files, otherwise it will ask to enter a name for each file. + @type keepFilename: bool + @param verifyDescription: Set to False to not proofread the description. + @type verifyDescription: bool + @param ignoreWarning: Set this to True to upload even if another file + would be overwritten or another mistake would be risked. Set it to + an array of warning codes to selectively ignore specific warnings. + @type ignoreWarning: bool or list + @param targetSite: Set the site to upload to. If target site is not + given it's taken from user-config.py. + @type targetSite: object + @param aborts: List of the warning types to abort upload on. Set to True + to abort on any warning. + @type aborts: bool or list + @param chunk_size: Upload the file in chunks (more overhead, but + restartable) specified in bytes. If no value is specified the file + will be uploaded as whole. + @type chunk_size: integer + @param always: Disables any input, requires that either ignoreWarning or + aborts are set to True and that the description is also set. It will + overwrite verifyDescription to False and keepFilename to True. + @type always: bool + + @deprecated: Using upload_image() is deprecated, use upload_file() with + file_url param instead + + """ + super(UploadRobot, self).__init__(**kwargs) + always = self.getOption('always') + if (always and ignoreWarning is not True and aborts is not True): + raise ValueError('When always is set to True, either ignoreWarning ' + 'or aborts must be set to True.') + if always and not description: + raise ValueError('When always is set to True the description must ' + 'be set.') + self.url = url + if isinstance(self.url, basestring): + pywikibot.warning("url as string is deprecated. " + "Use an iterable instead.") + self.urlEncoding = urlEncoding + self.description = description + self.useFilename = useFilename + self.keepFilename = keepFilename or always + self.verifyDescription = verifyDescription and not always + self.ignoreWarning = ignoreWarning + self.aborts = aborts + self.chunk_size = chunk_size + if config.upload_to_commons: + self.targetSite = targetSite or pywikibot.Site('commons', + 'commons') + else: + self.targetSite = targetSite or pywikibot.Site() + self.targetSite.login() + self.uploadByUrl = uploadByUrl + + @deprecated() + def urlOK(self): + """Return True if self.url is a URL or an existing local file.""" + return "://" in self.url or os.path.exists(self.url) + + def read_file_content(self, file_url=None): + """Return name of temp file in which remote file is saved.""" + if not file_url: + file_url = self.url + pywikibot.warning("file_url is not given. " + "Set to self.url by default.") + pywikibot.output(u'Reading file %s' % file_url) + resume = False + rlen = 0 + _contents = None + dt = 15 + uo = URLopener() + retrieved = False + + while not retrieved: + if resume: + pywikibot.output(u"Resume download...") + uo.addheader('Range', 'bytes=%s-' % rlen) + + infile = uo.open(file_url) + + if 'text/html' in infile.info().getheader('Content-Type'): + pywikibot.output(u"Couldn't download the image: " + "the requested URL was not found on server.") + return + + content_len = infile.info().getheader('Content-Length') + accept_ranges = infile.info().getheader('Accept-Ranges') == 'bytes' + + if resume: + _contents += infile.read() + else: + _contents = infile.read() + + infile.close() + retrieved = True + + if content_len: + rlen = len(_contents) + content_len = int(content_len) + if rlen < content_len: + retrieved = False + pywikibot.output( + u"Connection closed at byte %s (%s left)" + % (rlen, content_len)) + if accept_ranges and rlen > 0: + resume = True + pywikibot.output(u"Sleeping for %d seconds..." % dt) + time.sleep(dt) + if dt <= 60: + dt += 15 + elif dt < 360: + dt += 60 + else: + pywikibot.log( + u"WARNING: length check of retrieved data not possible.") + handle, tempname = tempfile.mkstemp() + with os.fdopen(handle, "wb") as t: + t.write(_contents) + return tempname + + def _handle_warning(self, warning): + """ + Return whether the warning cause an abort or be ignored. + + @param warning: The warning name + @type warning: str + @return: False if this warning should cause an abort, True if it should + be ignored or None if this warning has no default handler. + @rtype: bool or None + """ + if self.aborts is not True: + if warning in self.aborts: + return False + if self.ignoreWarning is True or (self.ignoreWarning is not False and + warning in self.ignoreWarning): + return True + return None if self.aborts is not True else False + + def _handle_warnings(self, warnings): + messages = '\n'.join('{0.code}: {0.info}'.format(warning) + for warning in sorted(warnings, + key=lambda w: w.code)) + if len(warnings) > 1: + messages = '\n' + messages + pywikibot.output('We got the following warning(s): ' + messages) + answer = True + for warning in warnings: + this_answer = self._handle_warning(warning.code) + if this_answer is False: + answer = False + break + elif this_answer is None: + answer = None + if answer is None: + answer = pywikibot.input_yn(u"Do you want to ignore?", + default=False, automatic_quit=False) + return answer + + def process_filename(self, file_url=None): + """Return base filename portion of file_url.""" + if not file_url: + file_url = self.url + pywikibot.warning("file_url is not given. " + "Set to self.url by default.") + + always = self.getOption('always') + # Isolate the pure name + filename = file_url + # Filename may be either a URL or a local file path + if "://" in filename: + # extract the path portion of the URL + filename = urlparse(filename).path + filename = os.path.basename(filename) + if self.useFilename: + filename = self.useFilename + if not self.keepFilename: + pywikibot.output( + u"The filename on the target wiki will default to: %s" + % filename) + assert not always + newfn = pywikibot.input( + u'Enter a better name, or press enter to accept:') + if newfn != "": + filename = newfn + # FIXME: these 2 belong somewhere else, presumably in family + # forbidden characters are handled by pywikibot/page.py + forbidden = ':*?/\' # to be extended + try: + allowed_formats = self.targetSite.siteinfo.get( + 'fileextensions', get_default=False) + except KeyError: + allowed_formats = [] + else: + allowed_formats = [item['ext'] for item in allowed_formats] + + # ask until it's valid + first_check = True + while True: + if not first_check: + if always: + filename = None + else: + filename = pywikibot.input('Enter a better name, or press ' + 'enter to skip the file:') + if not filename: + return None + first_check = False + ext = os.path.splitext(filename)[1].lower().strip('.') + # are any chars in forbidden also in filename? + invalid = set(forbidden) & set(filename) + if invalid: + c = "".join(invalid) + pywikibot.output( + 'Invalid character(s): %s. Please try again' % c) + continue + if allowed_formats and ext not in allowed_formats: + if always: + pywikibot.output('File format is not one of ' + '[{0}]'.format(' '.join(allowed_formats))) + continue + elif not pywikibot.input_yn( + u"File format is not one of [%s], but %s. Continue?" + % (u' '.join(allowed_formats), ext), + default=False, automatic_quit=False): + continue + potential_file_page = pywikibot.FilePage(self.targetSite, filename) + if potential_file_page.exists(): + overwrite = self._handle_warning('exists') + if overwrite is False: + pywikibot.output("File exists and you asked to abort. Skipping.") + return None + if potential_file_page.canBeEdited(): + if overwrite is None: + overwrite = not pywikibot.input_yn( + "File with name %s already exists. " + "Would you like to change the name? " + "(Otherwise file will be overwritten.)" + % filename, default=True, + automatic_quit=False) + if not overwrite: + continue + else: + break + else: + pywikibot.output(u"File with name %s already exists and " + "cannot be overwritten." % filename) + continue + else: + try: + if potential_file_page.fileIsShared(): + pywikibot.output(u"File with name %s already exists in shared " + "repository and cannot be overwritten." + % filename) + continue + else: + break + except pywikibot.NoPage: + break + + # A proper description for the submission. + # Empty descriptions are not accepted. + pywikibot.output(u'The suggested description is:\n%s' + % self.description) + + # Description must be set and verified + if not self.description: + self.verifyDescription = True + + while not self.description or self.verifyDescription: + if not self.description: + pywikibot.output(color_format( + '{lightred}It is not possible to upload a file ' + 'without a summary/description.{default}')) + + assert not always + # if no description, default is 'yes' + if pywikibot.input_yn( + u'Do you want to change this description?', + default=not self.description): + from pywikibot import editor as editarticle + editor = editarticle.TextEditor() + try: + newDescription = editor.edit(self.description) + except Exception as e: + pywikibot.error(e) + continue + # if user saved / didn't press Cancel + if newDescription: + self.description = newDescription + self.verifyDescription = False + + return filename + + def abort_on_warn(self, warn_code): + """Determine if the warning message should cause an abort.""" + if self.aborts is True: + return True + else: + return warn_code in self.aborts + + def ignore_on_warn(self, warn_code): + """Determine if the warning message should be ignored.""" + if self.ignoreWarning is True: + return True + else: + return warn_code in self.ignoreWarning + + @deprecated('UploadRobot.upload_file()') + def upload_image(self, debug=False): + """Upload image.""" + self.upload_file(self.url, debug) + + def upload_file(self, file_url, debug=False, _file_key=None, _offset=0): + """Upload the image at file_url to the target wiki. + + Return the filename that was used to upload the image. + If the upload fails, ask the user whether to try again or not. + If the user chooses not to retry, return null. + + """ + filename = self.process_filename(file_url) + if not filename: + return None + + site = self.targetSite + imagepage = pywikibot.FilePage(site, filename) # normalizes filename + imagepage.text = self.description + + pywikibot.output(u'Uploading file to %s via API...' % site) + + success = False + try: + if self.ignoreWarning is True: + apiIgnoreWarnings = True + else: + apiIgnoreWarnings = self._handle_warnings + if self.uploadByUrl: + success = site.upload(imagepage, source_url=file_url, + ignore_warnings=apiIgnoreWarnings, + _file_key=_file_key, _offset=_offset) + else: + if "://" in file_url: + temp = self.read_file_content(file_url) + else: + temp = file_url + success = site.upload(imagepage, source_filename=temp, + ignore_warnings=apiIgnoreWarnings, + chunk_size=self.chunk_size, + _file_key=_file_key, _offset=_offset) + + except pywikibot.data.api.APIError as error: + if error.code == u'uploaddisabled': + pywikibot.error("Upload error: Local file uploads are disabled on %s." + % site) + else: + pywikibot.error("Upload error: ", exc_info=True) + return None + except Exception: + pywikibot.error("Upload error: ", exc_info=True) + return None + else: + if success: + # No warning, upload complete. + pywikibot.output(u"Upload of %s successful." % filename) + return filename # data['filename'] + else: + pywikibot.output(u"Upload aborted.") + return None + + def run(self): + """Run bot.""" + # early check that upload is enabled + if self.targetSite.is_uploaddisabled(): + pywikibot.error( + "Upload error: Local file uploads are disabled on %s." + % self.targetSite) + return + + # early check that user has proper rights to upload + if "upload" not in self.targetSite.userinfo["rights"]: + pywikibot.error( + "User '%s' does not have upload rights on site %s." + % (self.targetSite.user(), self.targetSite)) + return + if isinstance(self.url, basestring): + return self.upload_file(self.url) + for file_url in self.url: + self.upload_file(file_url) diff --git a/scripts/data_ingestion.py b/scripts/data_ingestion.py index d8b0ca7..e1c9a51 100755 --- a/scripts/data_ingestion.py +++ b/scripts/data_ingestion.py @@ -38,9 +38,8 @@ from pywikibot import pagegenerators

from pywikibot.comms.http import fetch +from pywikibot.specialbots import UploadRobot from pywikibot.tools import deprecated, deprecated_args - -from scripts import upload

if sys.version_info[0] > 2: from urllib.parse import urlparse @@ -203,12 +202,12 @@ title = photo.getTitle(self.titlefmt) description = photo.getDescription(self.pagefmt)

- bot = upload.UploadRobot(url=photo.URL, - description=description, - useFilename=title, - keepFilename=True, - verifyDescription=False, - targetSite=self.site) + bot = UploadRobot(url=photo.URL, + description=description, + useFilename=title, + keepFilename=True, + verifyDescription=False, + targetSite=self.site) bot._contents = photo.downloadPhoto().getvalue() bot._retrieved = True bot.run() diff --git a/scripts/flickrripper.py b/scripts/flickrripper.py index d86f640..9782f91 100755 --- a/scripts/flickrripper.py +++ b/scripts/flickrripper.py @@ -25,7 +25,7 @@ """ # # (C) Multichill, 2009 -# (C) Pywikibot team, 2009-2014 +# (C) Pywikibot team, 2009-2016 # # Distributed under the terms of the MIT license. # @@ -59,7 +59,7 @@ from pywikibot import config, textlib from pywikibot.comms.http import fetch

-from scripts import upload +from pywikibot.specialbots import UploadRobot

try: from pywikibot.userinterfaces.gui import Tkdialog @@ -330,11 +330,11 @@ # Would be nice to check before I upload if the file is already at Commons # Not that important for this program, but maybe for derived programs if not skip: - bot = upload.UploadRobot(photoUrl, - description=newPhotoDescription, - useFilename=newFilename, - keepFilename=True, - verifyDescription=False) + bot = UploadRobot(photoUrl, + description=newPhotoDescription, + useFilename=newFilename, + keepFilename=True, + verifyDescription=False) bot.upload_image(debug=False) return 1 else: diff --git a/scripts/imagecopy.py b/scripts/imagecopy.py index ce82757..fc5b71e 100644 --- a/scripts/imagecopy.py +++ b/scripts/imagecopy.py @@ -95,9 +95,10 @@

from pywikibot import pagegenerators, config, i18n

+from pywikibot.specialbots import UploadRobot from pywikibot.tools import PY2

-from scripts import image, upload +from scripts import image

if not PY2: import tkinter as Tkinter @@ -292,11 +293,10 @@ 'added categories -->', '') CH += u'[[Category:' + self.category + u']]'

- bot = upload.UploadRobot(url=self.imagePage.fileUrl(), description=CH, - useFilename=self.newname, keepFilename=True, - verifyDescription=False, ignoreWarning=True, - targetSite=pywikibot.Site('commons', - 'commons')) + bot = UploadRobot(url=self.imagePage.fileUrl(), description=CH, + useFilename=self.newname, keepFilename=True, + verifyDescription=False, ignoreWarning=True, + targetSite=pywikibot.Site('commons', 'commons')) bot.run()

# Should check if the image actually was uploaded diff --git a/scripts/imagecopy_self.py b/scripts/imagecopy_self.py index 3b4eb1d..e55d945 100644 --- a/scripts/imagecopy_self.py +++ b/scripts/imagecopy_self.py @@ -63,9 +63,10 @@

from pywikibot import pagegenerators, i18n

+from pywikibot.specialbots import UploadRobot from pywikibot.tools import PY2

-from scripts import imagerecat, image, upload +from scripts import imagerecat, image

if not PY2: import tkinter as Tkinter @@ -890,13 +891,12 @@ """Work on a single image.""" cid = self.buildNewImageDescription(fields) pywikibot.output(cid) - bot = upload.UploadRobot(url=fields.get('imagepage').fileUrl(), - description=cid, - useFilename=fields.get('filename'), - keepFilename=True, verifyDescription=False, - ignoreWarning=True, - targetSite=pywikibot.Site('commons', - 'commons')) + bot = UploadRobot(url=fields.get('imagepage').fileUrl(), + description=cid, + useFilename=fields.get('filename'), + keepFilename=True, verifyDescription=False, + ignoreWarning=True, + targetSite=pywikibot.Site('commons', 'commons')) bot.run()

self.tagNowcommons(fields.get('imagepage'), fields.get('filename')) diff --git a/scripts/imageharvest.py b/scripts/imageharvest.py index 9026843..eee58ca 100644 --- a/scripts/imageharvest.py +++ b/scripts/imageharvest.py @@ -31,9 +31,8 @@

import pywikibot

+from pywikibot.specialbots import UploadRobot from pywikibot.tools import PY2 - -from scripts import upload

if not PY2: import urllib @@ -122,7 +121,7 @@ % (mysite.namespace(14), cat)) desc += "\r\n\r\n" + basicdesc + "\r\n\r\n" + \ "\r\n".join(categories) - uploadBot = upload.UploadRobot(image, description=desc) + uploadBot = UploadRobot(image, description=desc) uploadBot.run() elif answer == 's': break diff --git a/scripts/imagetransfer.py b/scripts/imagetransfer.py index 7c2a0fe..2eea20a 100755 --- a/scripts/imagetransfer.py +++ b/scripts/imagetransfer.py @@ -27,7 +27,7 @@ """ # # (C) Andre Engels, 2004 -# (C) Pywikibot team, 2004-2015 +# (C) Pywikibot team, 2004-2016 # # Distributed under the terms of the MIT license. # @@ -41,8 +41,8 @@ import pywikibot

from pywikibot import config, i18n, pagegenerators, textlib +from pywikibot.specialbots import UploadRobot

-from scripts import upload

nowCommonsTemplate = { 'ar': u'{{الآن كومنز|%s}}', @@ -168,12 +168,12 @@ description = '' print("Image description page is redirect.") else: - bot = upload.UploadRobot(url=url, description=description, - targetSite=self.targetSite, - urlEncoding=sourceSite.encoding(), - keepFilename=self.keep_name, - verifyDescription=not self.keep_name, - ignoreWarning=self.ignore_warning) + bot = UploadRobot(url=url, description=description, + targetSite=self.targetSite, + urlEncoding=sourceSite.encoding(), + keepFilename=self.keep_name, + verifyDescription=not self.keep_name, + ignoreWarning=self.ignore_warning) # try to upload targetFilename = bot.run() if targetFilename and self.targetSite.family.name == 'commons' and \ diff --git a/scripts/panoramiopicker.py b/scripts/panoramiopicker.py index 6dbc4a5..6108ff3 100644 --- a/scripts/panoramiopicker.py +++ b/scripts/panoramiopicker.py @@ -3,7 +3,7 @@ """Tool to copy a Panoramio set to Commons.""" # # (C) Multichill, 2010 -# (C) Pywikibot team, 2010-2015 +# (C) Pywikibot team, 2010-2016 # # Distributed under the terms of the MIT license. # @@ -24,9 +24,10 @@

from pywikibot import config

+from pywikibot.specialbots import UploadRobot from pywikibot.tools import PY2

-from scripts import imagerecat, upload +from scripts import imagerecat

if not PY2: from urllib.request import urlopen @@ -242,11 +243,11 @@ # Not that important for this program, but maybe for derived # programs if not skip: - bot = upload.UploadRobot(photoInfo.get(u'photo_file_url'), - description=newDescription, - useFilename=newFilename, - keepFilename=True, - verifyDescription=False, site=site) + bot = UploadRobot(photoInfo.get(u'photo_file_url'), + description=newDescription, + useFilename=newFilename, + keepFilename=True, + verifyDescription=False, site=site) bot.upload_image(debug=False) return 1 return 0 diff --git a/scripts/upload.py b/scripts/upload.py index 3f9935d..46b40e2 100755 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -47,7 +47,7 @@ """ # # (C) Rob W.W. Hooft, Andre Engels 2003-2004 -# (C) Pywikibot team, 2003-2015 +# (C) Pywikibot team, 2003-2016 # # Distributed under the terms of the MIT license. # @@ -55,447 +55,13 @@

__version__ = '$Id$' # - import math import os import re -import sys -import tempfile -import time

import pywikibot -import pywikibot.data.api - -from pywikibot import config - -from pywikibot.bot import suggest_help, BaseBot -from pywikibot.tools import ( - deprecated -) -from pywikibot.tools.formatter import color_format - -if sys.version_info[0] > 2: - from urllib.parse import urlparse - from urllib.request import URLopener - - basestring = (str,) -else: - from urllib import URLopener - from urlparse import urlparse - - -class UploadRobot(BaseBot): - - """Upload bot.""" - - def __init__(self, url, urlEncoding=None, description=u'', - useFilename=None, keepFilename=False, - verifyDescription=True, ignoreWarning=False, - targetSite=None, uploadByUrl=False, aborts=[], chunk_size=0, - **kwargs): - """ - Constructor. - - @param url: path to url or local file (deprecated), or list of urls or - paths to local files. - @type url: string (deprecated) or list - @param description: Description of file for its page. If multiple files - are uploading the same description is used for every file. - @type description: string - @param useFilename: Specify title of the file's page. If multiple - files are uploading it asks to change the name for second, third, - etc. files, otherwise the last file will overwrite the other. - @type useFilename: string - @param keepFilename: Set to True to keep original names of urls and - files, otherwise it will ask to enter a name for each file. - @type keepFilename: bool - @param verifyDescription: Set to False to not proofread the description. - @type verifyDescription: bool - @param ignoreWarning: Set this to True to upload even if another file - would be overwritten or another mistake would be risked. Set it to - an array of warning codes to selectively ignore specific warnings. - @type ignoreWarning: bool or list - @param targetSite: Set the site to upload to. If target site is not - given it's taken from user-config.py. - @type targetSite: object - @param aborts: List of the warning types to abort upload on. Set to True - to abort on any warning. - @type aborts: bool or list - @param chunk_size: Upload the file in chunks (more overhead, but - restartable) specified in bytes. If no value is specified the file - will be uploaded as whole. - @type chunk_size: integer - @param always: Disables any input, requires that either ignoreWarning or - aborts are set to True and that the description is also set. It will - overwrite verifyDescription to False and keepFilename to True. - @type always: bool - - @deprecated: Using upload_image() is deprecated, use upload_file() with - file_url param instead - - """ - super(UploadRobot, self).__init__(**kwargs) - always = self.getOption('always') - if (always and ignoreWarning is not True and aborts is not True): - raise ValueError('When always is set to True, either ignoreWarning ' - 'or aborts must be set to True.') - if always and not description: - raise ValueError('When always is set to True the description must ' - 'be set.') - self.url = url - if isinstance(self.url, basestring): - pywikibot.warning("url as string is deprecated. " - "Use an iterable instead.") - self.urlEncoding = urlEncoding - self.description = description - self.useFilename = useFilename - self.keepFilename = keepFilename or always - self.verifyDescription = verifyDescription and not always - self.ignoreWarning = ignoreWarning - self.aborts = aborts - self.chunk_size = chunk_size - if config.upload_to_commons: - self.targetSite = targetSite or pywikibot.Site('commons', - 'commons') - else: - self.targetSite = targetSite or pywikibot.Site() - self.targetSite.login() - self.uploadByUrl = uploadByUrl - - @deprecated() - def urlOK(self): - """Return True if self.url is a URL or an existing local file.""" - return "://" in self.url or os.path.exists(self.url) - - def read_file_content(self, file_url=None): - """Return name of temp file in which remote file is saved.""" - if not file_url: - file_url = self.url - pywikibot.warning("file_url is not given. " - "Set to self.url by default.") - pywikibot.output(u'Reading file %s' % file_url) - resume = False - rlen = 0 - _contents = None - dt = 15 - uo = URLopener() - retrieved = False - - while not retrieved: - if resume: - pywikibot.output(u"Resume download...") - uo.addheader('Range', 'bytes=%s-' % rlen) - - infile = uo.open(file_url) - - if 'text/html' in infile.info().getheader('Content-Type'): - pywikibot.output(u"Couldn't download the image: " - "the requested URL was not found on server.") - return - - content_len = infile.info().getheader('Content-Length') - accept_ranges = infile.info().getheader('Accept-Ranges') == 'bytes' - - if resume: - _contents += infile.read() - else: - _contents = infile.read() - - infile.close() - retrieved = True - - if content_len: - rlen = len(_contents) - content_len = int(content_len) - if rlen < content_len: - retrieved = False - pywikibot.output( - u"Connection closed at byte %s (%s left)" - % (rlen, content_len)) - if accept_ranges and rlen > 0: - resume = True - pywikibot.output(u"Sleeping for %d seconds..." % dt) - time.sleep(dt) - if dt <= 60: - dt += 15 - elif dt < 360: - dt += 60 - else: - pywikibot.log( - u"WARNING: length check of retrieved data not possible.") - handle, tempname = tempfile.mkstemp() - with os.fdopen(handle, "wb") as t: - t.write(_contents) - return tempname - - def _handle_warning(self, warning): - """ - Return whether the warning cause an abort or be ignored. - - @param warning: The warning name - @type warning: str - @return: False if this warning should cause an abort, True if it should - be ignored or None if this warning has no default handler. - @rtype: bool or None - """ - if self.aborts is not True: - if warning in self.aborts: - return False - if self.ignoreWarning is True or (self.ignoreWarning is not False and - warning in self.ignoreWarning): - return True - return None if self.aborts is not True else False - - def _handle_warnings(self, warnings): - messages = '\n'.join('{0.code}: {0.info}'.format(warning) - for warning in sorted(warnings, - key=lambda w: w.code)) - if len(warnings) > 1: - messages = '\n' + messages - pywikibot.output('We got the following warning(s): ' + messages) - answer = True - for warning in warnings: - this_answer = self._handle_warning(warning.code) - if this_answer is False: - answer = False - break - elif this_answer is None: - answer = None - if answer is None: - answer = pywikibot.input_yn(u"Do you want to ignore?", - default=False, automatic_quit=False) - return answer - - def process_filename(self, file_url=None): - """Return base filename portion of file_url.""" - if not file_url: - file_url = self.url - pywikibot.warning("file_url is not given. " - "Set to self.url by default.") - - always = self.getOption('always') - # Isolate the pure name - filename = file_url - # Filename may be either a URL or a local file path - if "://" in filename: - # extract the path portion of the URL - filename = urlparse(filename).path - filename = os.path.basename(filename) - if self.useFilename: - filename = self.useFilename - if not self.keepFilename: - pywikibot.output( - u"The filename on the target wiki will default to: %s" - % filename) - assert not always - newfn = pywikibot.input( - u'Enter a better name, or press enter to accept:') - if newfn != "": - filename = newfn - # FIXME: these 2 belong somewhere else, presumably in family - # forbidden characters are handled by pywikibot/page.py - forbidden = ':*?/\' # to be extended - try: - allowed_formats = self.targetSite.siteinfo.get( - 'fileextensions', get_default=False) - except KeyError: - allowed_formats = [] - else: - allowed_formats = [item['ext'] for item in allowed_formats] - - # ask until it's valid - first_check = True - while True: - if not first_check: - if always: - filename = None - else: - filename = pywikibot.input('Enter a better name, or press ' - 'enter to skip the file:') - if not filename: - return None - first_check = False - ext = os.path.splitext(filename)[1].lower().strip('.') - # are any chars in forbidden also in filename? - invalid = set(forbidden) & set(filename) - if invalid: - c = "".join(invalid) - pywikibot.output( - 'Invalid character(s): %s. Please try again' % c) - continue - if allowed_formats and ext not in allowed_formats: - if always: - pywikibot.output('File format is not one of ' - '[{0}]'.format(' '.join(allowed_formats))) - continue - elif not pywikibot.input_yn( - u"File format is not one of [%s], but %s. Continue?" - % (u' '.join(allowed_formats), ext), - default=False, automatic_quit=False): - continue - potential_file_page = pywikibot.FilePage(self.targetSite, filename) - if potential_file_page.exists(): - overwrite = self._handle_warning('exists') - if overwrite is False: - pywikibot.output("File exists and you asked to abort. Skipping.") - return None - if potential_file_page.canBeEdited(): - if overwrite is None: - overwrite = not pywikibot.input_yn( - "File with name %s already exists. " - "Would you like to change the name? " - "(Otherwise file will be overwritten.)" - % filename, default=True, - automatic_quit=False) - if not overwrite: - continue - else: - break - else: - pywikibot.output(u"File with name %s already exists and " - "cannot be overwritten." % filename) - continue - else: - try: - if potential_file_page.fileIsShared(): - pywikibot.output(u"File with name %s already exists in shared " - "repository and cannot be overwritten." - % filename) - continue - else: - break - except pywikibot.NoPage: - break - - # A proper description for the submission. - # Empty descriptions are not accepted. - pywikibot.output(u'The suggested description is:\n%s' - % self.description) - - # Description must be set and verified - if not self.description: - self.verifyDescription = True - - while not self.description or self.verifyDescription: - if not self.description: - pywikibot.output(color_format( - '{lightred}It is not possible to upload a file ' - 'without a summary/description.{default}')) - - assert not always - # if no description, default is 'yes' - if pywikibot.input_yn( - u'Do you want to change this description?', - default=not self.description): - from pywikibot import editor as editarticle - editor = editarticle.TextEditor() - try: - newDescription = editor.edit(self.description) - except Exception as e: - pywikibot.error(e) - continue - # if user saved / didn't press Cancel - if newDescription: - self.description = newDescription - self.verifyDescription = False - - return filename - - def abort_on_warn(self, warn_code): - """Determine if the warning message should cause an abort.""" - if self.aborts is True: - return True - else: - return warn_code in self.aborts - - def ignore_on_warn(self, warn_code): - """Determine if the warning message should be ignored.""" - if self.ignoreWarning is True: - return True - else: - return warn_code in self.ignoreWarning - - @deprecated('UploadRobot.upload_file()') - def upload_image(self, debug=False): - """Upload image.""" - self.upload_file(self.url, debug) - - def upload_file(self, file_url, debug=False, _file_key=None, _offset=0): - """Upload the image at file_url to the target wiki. - - Return the filename that was used to upload the image. - If the upload fails, ask the user whether to try again or not. - If the user chooses not to retry, return null. - - """ - filename = self.process_filename(file_url) - if not filename: - return None - - site = self.targetSite - imagepage = pywikibot.FilePage(site, filename) # normalizes filename - imagepage.text = self.description - - pywikibot.output(u'Uploading file to %s via API...' % site) - - success = False - try: - if self.ignoreWarning is True: - apiIgnoreWarnings = True - else: - apiIgnoreWarnings = self._handle_warnings - if self.uploadByUrl: - success = site.upload(imagepage, source_url=file_url, - ignore_warnings=apiIgnoreWarnings, - _file_key=_file_key, _offset=_offset) - else: - if "://" in file_url: - temp = self.read_file_content(file_url) - else: - temp = file_url - success = site.upload(imagepage, source_filename=temp, - ignore_warnings=apiIgnoreWarnings, - chunk_size=self.chunk_size, - _file_key=_file_key, _offset=_offset) - - except pywikibot.data.api.APIError as error: - if error.code == u'uploaddisabled': - pywikibot.error("Upload error: Local file uploads are disabled on %s." - % site) - else: - pywikibot.error("Upload error: ", exc_info=True) - return None - except Exception: - pywikibot.error("Upload error: ", exc_info=True) - return None - else: - if success: - # No warning, upload complete. - pywikibot.output(u"Upload of %s successful." % filename) - return filename # data['filename'] - else: - pywikibot.output(u"Upload aborted.") - return None - - def run(self): - """Run bot.""" - # early check that upload is enabled - if self.targetSite.is_uploaddisabled(): - pywikibot.error( - "Upload error: Local file uploads are disabled on %s." - % self.targetSite) - return - - # early check that user has proper rights to upload - if "upload" not in self.targetSite.userinfo["rights"]: - pywikibot.error( - "User '%s' does not have upload rights on site %s." - % (self.targetSite.user(), self.targetSite)) - return - if isinstance(self.url, basestring): - return self.upload_file(self.url) - for file_url in self.url: - self.upload_file(file_url) +from pywikibot.bot import suggest_help +from pywikibot.specialbots import UploadRobot

def main(*args):