jenkins-bot has submitted this change and it was merged.

Change subject: Added a list of urls upload support ......................................................................

Added a list of urls upload support

Test images are now stored in data/images. Use _images_dir as the path variable. upload_image() is now deprecated, use upload_file(file_url) instead. Bot's parameter url is deprecated to be string, now it's an iterable. upload.py does not use urlOK() method anymore. The check is moved to main(). To process_filename() added a check whether there is already a page with the same title. Docstring for __init__() is extended.

While T85708 is not fixed, try to specify filenames with correct extention.

Bug: T57088 Change-Id: I13d469b3ddcac78b02543efe53868a8672966ec0 --- M scripts/upload.py M tests/__init__.py A tests/data/images/1rightarrow.png R tests/data/images/MP_sounds.png M tests/data_ingestion_tests.py M tests/dry_api_tests.py M tests/upload_tests.py A tests/uploadbot_tests.py 8 files changed, 232 insertions(+), 74 deletions(-)

Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified

diff --git a/scripts/upload.py b/scripts/upload.py index 91e1976..e3daa44 100755 --- a/scripts/upload.py +++ b/scripts/upload.py @@ -23,17 +23,17 @@ 'Mi': Mebibytes (1024x1024 B) The suffixes are case insensitive.

-If any other arguments are given, the first is the URL or filename to upload, -and the rest is a proposed description to go with the upload. If none of these -are given, the user is asked for the file or URL to upload. The bot will then -upload the image to the wiki. +If any other arguments are given, the first is either URL, filename or directory +to upload, and the rest is a proposed description to go with the upload. If none +of these are given, the user is asked for the directory, file or URL to upload. +The bot will then upload the image to the wiki.

-The script will ask for the location of an image, if not given as a parameter, +The script will ask for the location of an image(s), if not given as a parameter, and for a description. """ # # (C) Rob W.W. Hooft, Andre Engels 2003-2004 -# (C) Pywikibot team, 2003-2014 +# (C) Pywikibot team, 2003-2015 # # Distributed under the terms of the MIT license. # @@ -50,6 +50,9 @@ import pywikibot import pywikibot.data.api from pywikibot import config +from pywikibot.tools import ( + deprecated +)

if sys.version_info[0] > 2: from urllib.parse import urlparse @@ -70,13 +73,44 @@ """ Constructor.

- @param ignoreWarning: Set this to True if you want to upload even if - another file would be overwritten or another mistake would be - risked. You can also set it to an array of warning codes to - selectively ignore specific warnings. + @param url: path to url or local file (deprecated), or list of urls or + paths to local files. + @type url: string (deprecated) or list + @param description: Description of file for its page. If multiple files + are uploading the same description is used for every file. + @type description: string + @param useFilename: Specify title of the file's page. If multiple + files are uploading it asks to change the name for second, third, + etc. files, otherwise the last file will overwrite the other. + @type useFilename: string + @param keepFilename: Set to True to keep original names of urls and + files, otherwise it will ask to enter a name for each file. + @type keepFilename: bool + @param verifyDescription: Set to False to not proofread the description. + @type verifyDescription: bool + @param ignoreWarning: Set this to True to upload even if another file + would be overwritten or another mistake would be risked. Set it to + an array of warning codes to selectively ignore specific warnings. + @type ignoreWarning: bool or list + @param targetSite: Set the site to upload to. If target site is not + given it's taken from user-config.py. + @type targetSite: object + @param aborts: List of the warning types to abort upload on. Set to True + to abort on any warning. + @type aborts: bool or list + @param chunk_size: Upload the file in chunks (more overhead, but + restartable) specified in bytes. If no value is specified the file + will be uploaded as whole. + @type chunk_size: integer + + @deprecated: Using upload_image() is deprecated, use upload_file() with + file_url param instead

""" self.url = url + if isinstance(self.url, basestring): + pywikibot.warning("url as string is deprecated. " + "Use an iterable instead.") self.urlEncoding = urlEncoding self.description = description self.useFilename = useFilename @@ -90,16 +124,21 @@ 'commons') else: self.targetSite = targetSite or pywikibot.Site() - self.targetSite.forceLogin() + self.targetSite.login() self.uploadByUrl = uploadByUrl

+ @deprecated() def urlOK(self): - """Return True if self.url is an URL or an existing local file.""" + """Return True if self.url is a URL or an existing local file.""" return "://" in self.url or os.path.exists(self.url)

- def read_file_content(self): + def read_file_content(self, file_url=None): """Return name of temp file in which remote file is saved.""" - pywikibot.output(u'Reading file %s' % self.url) + if not file_url: + file_url = self.url + pywikibot.warning("file_url is not given. " + "Set to self.url by default.") + pywikibot.output(u'Reading file %s' % file_url) resume = False rlen = 0 _contents = None @@ -112,7 +151,7 @@ pywikibot.output(u"Resume download...") uo.addheader('Range', 'bytes=%s-' % rlen)

- infile = uo.open(self.url) + infile = uo.open(file_url)

if 'text/html' in infile.info().getheader('Content-Type'): pywikibot.output(u"Couldn't download the image: " @@ -150,56 +189,89 @@ pywikibot.log( u"WARNING: length check of retrieved data not possible.") handle, tempname = tempfile.mkstemp() - t = os.fdopen(handle, "wb") - t.write(_contents) + with os.fdopen(handle, "wb") as t: + t.write(_contents) t.close() return tempname

- def process_filename(self): - """Return base filename portion of self.url.""" + def process_filename(self, file_url=None): + """Return base filename portion of file_url.""" + if not file_url: + file_url = self.url + pywikibot.warning("file_url is not given. " + "Set to self.url by default.") + # Isolate the pure name - filename = self.url - # Filename may be either a local file path or a URL + filename = file_url + # Filename may be either a URL or a local file path if "://" in filename: # extract the path portion of the URL filename = urlparse(filename).path filename = os.path.basename(filename) - if self.useFilename: filename = self.useFilename if not self.keepFilename: pywikibot.output( u"The filename on the target wiki will default to: %s" % filename) - # FIXME: these 2 belong somewhere else, presumably in family - forbidden = '/' # to be extended - allowed_formats = (u'gif', u'jpg', u'jpeg', u'mid', u'midi', - u'ogg', u'png', u'svg', u'xcf', u'djvu', - u'ogv', u'oga', u'tif', u'tiff') - # ask until it's valid - while True: - newfn = pywikibot.input( - u'Enter a better name, or press enter to accept:') - if newfn == "": - newfn = filename - break - ext = os.path.splitext(newfn)[1].lower().strip('.') - # are any chars in forbidden also in newfn? - invalid = set(forbidden) & set(newfn) - if invalid: - c = "".join(invalid) - pywikibot.output( - 'Invalid character(s): %s. Please try again' % c) - continue - if ext not in allowed_formats: - if not pywikibot.input_yn( - u"File format is not one of [%s], but %s. Continue?" - % (u' '.join(allowed_formats), ext), - default=False, automatic_quit=False): - continue - break - if newfn != '': + newfn = pywikibot.input( + u'Enter a better name, or press enter to accept:') + if newfn != "": filename = newfn + # FIXME: these 2 belong somewhere else, presumably in family + # forbidden characters are handled by pywikibot/page.py + forbidden = ':*?/\' # to be extended + allowed_formats = (u'gif', u'jpg', u'jpeg', u'mid', u'midi', + u'ogg', u'png', u'svg', u'xcf', u'djvu', + u'ogv', u'oga', u'tif', u'tiff') + # ask until it's valid + first_check = True + while True: + if not first_check: + filename = pywikibot.input(u'Enter a better name, ' + 'or press enter to skip:') + if not filename: + return None + first_check = False + ext = os.path.splitext(filename)[1].lower().strip('.') + # are any chars in forbidden also in filename? + invalid = set(forbidden) & set(filename) + if invalid: + c = "".join(invalid) + pywikibot.output( + 'Invalid character(s): %s. Please try again' % c) + continue + if ext not in allowed_formats: + if not pywikibot.input_yn( + u"File format is not one of [%s], but %s. Continue?" + % (u' '.join(allowed_formats), ext), + default=False, automatic_quit=False): + continue + potential_file_page = pywikibot.FilePage(self.targetSite, filename) + if potential_file_page.exists(): + if potential_file_page.canBeEdited(): + if pywikibot.input_yn(u"File with name %s already exists. " + "Would you like to change the name? " + "(Otherwise file will be overwritten.)" + % filename, default=True, + automatic_quit=False): + continue + else: + break + else: + pywikibot.output(u"File with name %s already exists and " + "cannot be overwritten." % filename) + continue + else: + try: + if potential_file_page.fileIsShared(): + pywikibot.output(u"File with name %s already exists in shared " + "repository and cannot be overwritten." + % filename) + continue + except pywikibot.NoPage: + break + # A proper description for the submission. # Empty descriptions are not accepted. pywikibot.output(u'The suggested description is:\n%s' @@ -247,34 +319,41 @@ else: return warn_code in self.ignoreWarning

+ @deprecated('UploadRobot.upload_file()') def upload_image(self, debug=False): - """Upload the image at self.url to the target wiki. + """Upload image.""" + self.upload_file(self.url, debug) + + def upload_file(self, file_url, debug=False): + """Upload the image at file_url to the target wiki.

Return the filename that was used to upload the image. If the upload fails, ask the user whether to try again or not. If the user chooses not to retry, return null.

""" - filename = self.process_filename() + filename = self.process_filename(file_url) + if not filename: + return None

site = self.targetSite imagepage = pywikibot.FilePage(site, filename) # normalizes filename imagepage.text = self.description

- pywikibot.output(u'Uploading file to %s via API....' % site) + pywikibot.output(u'Uploading file to %s via API...' % site)

try: apiIgnoreWarnings = False if self.ignoreWarning is True: apiIgnoreWarnings = True if self.uploadByUrl: - site.upload(imagepage, source_url=self.url, + site.upload(imagepage, source_url=file_url, ignore_warnings=apiIgnoreWarnings) else: - if "://" in self.url: - temp = self.read_file_content() + if "://" in file_url: + temp = self.read_file_content(file_url) else: - temp = self.url + temp = file_url site.upload(imagepage, source_filename=temp, ignore_warnings=apiIgnoreWarnings, chunk_size=self.chunk_size) @@ -307,7 +386,7 @@

else: # No warning, upload complete. - pywikibot.output(u"Upload successful.") + pywikibot.output(u"Upload of %s successful." % filename) return filename # data['filename']

def run(self): @@ -325,14 +404,10 @@ "User '%s' does not have upload rights on site %s." % (self.targetSite.user(), self.targetSite)) return - - while not self.urlOK(): - if not self.url: - pywikibot.output(u'No input filename given') - else: - pywikibot.output(u'Invalid input filename given. Try again.') - self.url = pywikibot.input(u'File or URL where image is now:') - return self.upload_image() + if isinstance(self.url, basestring): + return self.upload_file(self.url) + for file_url in self.url: + self.upload_file(file_url)

def main(*args): @@ -403,6 +478,20 @@ url = arg else: description.append(arg) + while not ("://" in url or os.path.exists(url)): + if not url: + pywikibot.output(u'No input filename given.') + else: + pywikibot.output(u'Invalid input filename given. Try again.') + url = pywikibot.input(u'URL, file or directory where files are now:') + if os.path.isdir(url): + file_list = [] + for directory_info in os.walk(url): + for dir_file in directory_info[2]: + file_list.append(os.path.join(directory_info[0], dir_file)) + url = file_list + else: + url = [url] description = u' '.join(description) bot = UploadRobot(url, description=description, useFilename=useFilename, keepFilename=keepFilename, diff --git a/tests/__init__.py b/tests/__init__.py index cd5ef45..df252b7 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -51,6 +51,7 @@ _tests_dir = os.path.split(__file__)[0] _cache_dir = os.path.join(_tests_dir, 'apicache') _data_dir = os.path.join(_tests_dir, 'data') +_images_dir = os.path.join(_data_dir, 'images')

# Find the root directory of the checkout _root_dir = os.path.split(_tests_dir)[0] diff --git a/tests/data/images/1rightarrow.png b/tests/data/images/1rightarrow.png new file mode 100644 index 0000000..fbce893 --- /dev/null +++ b/tests/data/images/1rightarrow.png Binary files differ diff --git a/tests/data/MP_sounds.png b/tests/data/images/MP_sounds.png similarity index 100% rename from tests/data/MP_sounds.png rename to tests/data/images/MP_sounds.png Binary files differ diff --git a/tests/data_ingestion_tests.py b/tests/data_ingestion_tests.py index 312bea0..b62a999 100644 --- a/tests/data_ingestion_tests.py +++ b/tests/data_ingestion_tests.py @@ -6,6 +6,7 @@

import os from tests import _data_dir +from tests import _images_dir from tests.aspects import unittest, TestCase from scripts import data_ingestion

@@ -37,7 +38,7 @@

def test_downloadPhoto(self): """Test download from http://upload.wikimedia.org/.""" - with open(os.path.join(_data_dir, 'MP_sounds.png'), 'rb') as f: + with open(os.path.join(_images_dir, 'MP_sounds.png'), 'rb') as f: self.assertEqual(f.read(), self.obj.downloadPhoto().read())

def test_findDuplicateImages(self): diff --git a/tests/dry_api_tests.py b/tests/dry_api_tests.py index 3b3afe4..75a384a 100644 --- a/tests/dry_api_tests.py +++ b/tests/dry_api_tests.py @@ -20,7 +20,7 @@ ) from pywikibot.family import Family

-from tests import _data_dir +from tests import _images_dir from tests.utils import DummySiteinfo from tests.aspects import unittest, TestCase, DefaultDrySiteTestCase

@@ -188,7 +188,7 @@

def test_mime_file_payload(self): """Test Request._generate_MIME_part loads binary as binary.""" - local_filename = os.path.join(_data_dir, 'MP_sounds.png') + local_filename = os.path.join(_images_dir, 'MP_sounds.png') with open(local_filename, 'rb') as f: file_content = f.read() submsg = Request._generate_MIME_part( @@ -197,7 +197,7 @@ self.assertEqual(file_content, submsg.get_payload(decode=True))

def test_mime_file_container(self): - local_filename = os.path.join(_data_dir, 'MP_sounds.png') + local_filename = os.path.join(_images_dir, 'MP_sounds.png') with open(local_filename, 'rb') as f: file_content = f.read() body = Request._build_mime_request({}, { @@ -215,7 +215,7 @@ """Test Request object prepared to upload.""" req = Request(site=self.get_site(), action="upload", file='MP_sounds.png', mime=True, - filename=os.path.join(_data_dir, 'MP_sounds.png')) + filename=os.path.join(_images_dir, 'MP_sounds.png')) self.assertEqual(req.mime, True)

diff --git a/tests/upload_tests.py b/tests/upload_tests.py index a7f978f..628362b 100644 --- a/tests/upload_tests.py +++ b/tests/upload_tests.py @@ -15,7 +15,7 @@

import pywikibot

-from tests import _data_dir +from tests import _images_dir from tests.aspects import unittest, TestCase

@@ -31,7 +31,7 @@ def test_png(self): """Test uploading a png using Site.upload.""" page = pywikibot.FilePage(self.site, 'MP_sounds-pwb.png') - local_filename = os.path.join(_data_dir, 'MP_sounds.png') + local_filename = os.path.join(_images_dir, 'MP_sounds.png') self.site.upload(page, source_filename=local_filename, comment='pywikibot test', ignore_warnings=True) @@ -39,7 +39,7 @@ def test_png_chunked(self): """Test uploading a png in two chunks using Site.upload.""" page = pywikibot.FilePage(self.site, 'MP_sounds-pwb-chunked.png') - local_filename = os.path.join(_data_dir, 'MP_sounds.png') + local_filename = os.path.join(_images_dir, 'MP_sounds.png') self.site.upload(page, source_filename=local_filename, comment='pywikibot test', ignore_warnings=True, chunk_size=1024) diff --git a/tests/uploadbot_tests.py b/tests/uploadbot_tests.py new file mode 100644 index 0000000..a84667c --- /dev/null +++ b/tests/uploadbot_tests.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +""" +UploadRobot test. + +These tests write to the wiki. +""" +# +# (C) Pywikibot team, 2015 +# +# Distributed under the terms of the MIT license. +# +__version__ = '$Id$' +# + +import os + +from scripts import upload +from tests import _images_dir +from tests.aspects import unittest, TestCase + + +class TestUploadbot(TestCase): + + """Test cases for upload.""" + + write = True + + family = 'wikipedia' + code = 'test' + + def test_png_list(self): + """Test uploading a list of pngs using upload.py.""" + image_list = [] + for directory_info in os.walk(_images_dir): + for dir_file in directory_info[2]: + image_list.append(os.path.join(directory_info[0], dir_file)) + bot = upload.UploadRobot(url=image_list, + description="pywikibot upload.py script test", + useFilename=None, keepFilename=True, + verifyDescription=True, aborts=set(), + ignoreWarning=True, targetSite=self.get_site()) + bot.run() + + def test_png(self): + """Test uploading a png using upload.py.""" + bot = upload.UploadRobot(url=[os.path.join(_images_dir, "MP_sounds.png")], + description="pywikibot upload.py script test", + useFilename=None, keepFilename=True, + verifyDescription=True, aborts=set(), + ignoreWarning=True, targetSite=self.get_site()) + bot.run() + + def test_png_url(self): + """Test uploading a png from url using upload.py.""" + bot = upload.UploadRobot(url=['https://upload.wikimedia.org/wikipedia/commons/f/fc/MP_sounds.png'], + description="pywikibot upload.py script test", + useFilename=None, keepFilename=True, + verifyDescription=True, aborts=set(), + ignoreWarning=True, targetSite=self.get_site()) + bot.run() + + +if __name__ == '__main__': + try: + unittest.main() + except SystemExit: + pass