jenkins-bot has submitted this change and it was merged.
Change subject: listpages.py: Add option to save page in local file
......................................................................
listpages.py: Add option to save page in local file
Provide the option to save Page to a local file.
File has the same name as the page.
File encoding can be specified and it defaults
to config.textfile_encoding.
Directory can be optionally specified and created.
Change-Id: I03f13f605301c84583c7f1f9b3916ba357e08b9d
---
M scripts/listpages.py
1 file changed, 63 insertions(+), 18 deletions(-)
Approvals:
XZise: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/listpages.py b/scripts/listpages.py
index 0580a54..72cfcb5 100644
--- a/scripts/listpages.py
+++ b/scripts/listpages.py
@@ -2,7 +2,8 @@
r"""
Print a list of pages, as defined by page generator parameters.
-Optionally, it also prints page content to STDOUT.
+Optionally, it also prints page content to STDOUT or save it to a file
+in the current directory.
These parameters are supported to specify which pages titles to print:
@@ -34,30 +35,39 @@
num is the sequential number of the listed page.
--outputlang Language for translation of namespaces
+-outputlang Language for translation of namespaces.
-notitle Page title is not printed.
-get Page content is printed.
+-save Save Page content to a file named as page.title(as_filename=True).
+ Directory can be set with -dir:dir_name
+ If no dir is specified, current direcory will be used.
+
+-encode File encoding can be specified with '-encode:name' (name must be a
+ valid python encoding: utf-8, etc.).
+ If not specified, it defaults to config.textfile_encoding.
+
Custom format can be applied to the following items extrapolated from a
page object:
- site: obtained from page._link._site
+ site: obtained from page._link._site.
- title: obtained from page._link._title
+ title: obtained from page._link._title.
- loc_title: obtained from page._link.canonical_title()
+ loc_title: obtained from page._link.canonical_title().
- can_title: obtained from page._link.ns_title()
+ can_title: obtained from page._link.ns_title().
based either the canonical namespace name or on the namespace name
in the language specified by the -trans param;
a default value '******' will be used if no ns is found.
- onsite: obtained from pywikibot.Site(outputlang, self.site.family)
+ onsite: obtained from pywikibot.Site(outputlang, self.site.family).
- trs_title: obtained from page._link.ns_title(onsite=onsite)
+ trs_title: obtained from page._link.ns_title(onsite=onsite).
+ If selected format requires trs_title, outputlang must be set.
¶ms;
@@ -70,8 +80,10 @@
__version__ = '$Id$'
#
+import os
import pywikibot
+from pywikibot import config2 as config
from pywikibot.pagegenerators import GeneratorFactory, parameterHelp
docuReplacements = {'¶ms;': parameterHelp}
@@ -103,12 +115,12 @@
@param outputlang: language code in which namespace before title should
be translated.
- Page namespace will be searched in Site(outputlang, page.site.family)
+ Page ns will be searched in Site(outputlang, page.site.family)
and, if found, its custom name will be used in page.title().
@type outputlang: str or None, if no translation is wanted.
- @param default: default string to be used if no corresponding namespace
- is found when outputlang is not None.
+ @param default: default string to be used if no corresponding
+ namespace is found when outputlang is not None.
"""
self.site = page._link.site
@@ -117,7 +129,7 @@
self.can_title = page._link.ns_title()
self.outputlang = outputlang
if outputlang is not None:
- # Cache onsite in case of tranlations.
+ # Cache onsite in case of translations.
if not hasattr(self, "onsite"):
self.onsite = pywikibot.Site(outputlang, self.site.family)
try:
@@ -155,6 +167,8 @@
fmt = '1'
outputlang = None
page_get = False
+ base_dir = None
+ encoding = config.textfile_encoding
# Process global args and prepare generator args parser
local_args = pywikibot.handle_args(args)
@@ -163,15 +177,39 @@
for arg in local_args:
if arg == '-notitle':
notitle = True
- elif arg.startswith("-format:"):
- fmt = arg[len("-format:"):]
+ elif arg.startswith('-format:'):
+ fmt = arg[len('-format:'):]
fmt = fmt.replace(u'\\03{{', u'\03{{')
- elif arg.startswith("-outputlang:"):
- outputlang = arg[len("-outputlang:"):]
+ elif arg.startswith('-outputlang:'):
+ outputlang = arg[len('-outputlang:'):]
elif arg == '-get':
page_get = True
+ elif arg.startswith('-save'):
+ base_dir = arg.partition(':')[2] or '.'
+ elif arg.startswith('-encode:'):
+ encoding = arg.partition(':')[2]
else:
genFactory.handleArg(arg)
+
+ if base_dir:
+ base_dir = os.path.expanduser(base_dir)
+ if not os.path.isabs(base_dir):
+ base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
+
+ if not os.path.exists(base_dir):
+ pywikibot.output(u'Directory "%s" does not exist.' % base_dir)
+ choice = pywikibot.input_yn(
+ u'Do you want to create it ("No" to continue without saving)?')
+ if choice:
+ os.makedirs(base_dir, mode=0o744)
+ else:
+ base_dir = None
+ elif not os.path.isdir(base_dir):
+ # base_dir is a file.
+ pywikibot.warning(u'Not a directory: "%s"\n'
+ u'Skipping saving ...'
+ % base_dir)
+ base_dir = None
gen = genFactory.getCombinedGenerator()
if gen:
@@ -180,8 +218,15 @@
page_fmt = Formatter(page, outputlang)
pywikibot.stdout(page_fmt.output(num=i, fmt=fmt))
if page_get:
- # TODO: catch exceptions
- pywikibot.output(page.text, toStdout=True)
+ try:
+ pywikibot.output(page.text, toStdout=True)
+ except pywikibot.Error as err:
+ pywikibot.output(err)
+ if base_dir:
+ filename = os.path.join(base_dir, page.title(as_filename=True))
+ pywikibot.output(u'Saving %s to %s' % (page.title(), filename))
+ with open(filename, mode='wb') as f:
+ f.write(page.text.encode(encoding))
else:
pywikibot.showHelp()
--
To view, visit https://gerrit.wikimedia.org/r/182461
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I03f13f605301c84583c7f1f9b3916ba357e08b9d
Gerrit-PatchSet: 6
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>
Build Update for wikimedia/pywikibot-core
-------------------------------------
Build: #1886
Status: Errored
Duration: 44 minutes and 57 seconds
Commit: e443343 (master)
Author: jenkins-bot
Message: Updated pywikibot/core
Project: pywikibot/i18n f950a0e13ea746bb01da0e157b92d517d6efbd4f
Add grunt test to validate i18n JSON
All i18n JSON files were invalid due to missing metadata blocks.
weblinkchecker has missing messages in many languages.
Bug: T85335
Bug: T85336
Change-Id: Ieb1f96889848332a7163a7c1a1070e99c560dc6e
View the changeset: https://github.com/wikimedia/pywikibot-core/compare/5625fe60eafc...e4433434…
View the full build log and details: https://travis-ci.org/wikimedia/pywikibot-core/builds/45763062
--
You can configure recipients for build notifications in your .travis.yml file. See http://docs.travis-ci.com/user/notifications
jenkins-bot has submitted this change and it was merged.
Change subject: Backport ca and th L10N from core to compat
......................................................................
Backport ca and th L10N from core to compat
This patch backports L10N for ca and th code and doc changes
from core to compat.
Set fallback=False for i18n.translate() because this is
L10N and not i18n here.
Change-Id: Ib880d311c97938ca75575a85f120200db387ae9f
---
M noreferences.py
1 file changed, 60 insertions(+), 23 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/noreferences.py b/noreferences.py
index 8f56fd5..8ffa0a5 100644
--- a/noreferences.py
+++ b/noreferences.py
@@ -1,8 +1,9 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
-
"""
-This script goes over multiple pages, searches for pages where <references />
+This script adds a missing references section to pages.
+
+It goes over multiple pages, searches for pages where <references />
is missing although a <ref> tag is present, and in that case adds a new
references section.
@@ -11,7 +12,7 @@
¶ms;
-xml Retrieve information from a local XML dump (pages-articles
- or pages-meta-current, see http://download.wikimedia.org).
+ or pages-meta-current, see https://download.wikimedia.org).
Argument can also be given as "-xml:filename".
-namespace:n Number or name of namespace to process. The parameter can be
@@ -34,7 +35,7 @@
a list of affected articles
"""
#
-# (C) Pywikibot team, 2007-2013
+# (C) Pywikibot team, 2007-2014
#
# Distributed under the terms of the MIT license.
#
@@ -64,6 +65,13 @@
u'وصلات خارجية',
u'انظر أيضا',
u'ملاحظات'
+ ],
+ 'ca': [
+ u'Bibliografia',
+ u'Bibliografia complementària',
+ u'Vegeu també',
+ u'Enllaços externs',
+ u'Enllaços',
],
'cs': [
u'Externí odkazy',
@@ -192,6 +200,12 @@
u'Przipisy',
u'Připisy',
],
+ 'th': [
+ u'อ่านเพิ่มเติม',
+ u'แหล่งข้อมูลอื่น',
+ u'ดูเพิ่ม',
+ u'หมายเหตุ',
+ ],
'zh': [
u'外部链接',
u'外部連结',
@@ -213,6 +227,9 @@
u'مصادر ومراجع',
u'المراجع والمصادر',
u'المصادر والمراجع',
+ ],
+ 'ca': [
+ u'Referències',
],
'cs': [
u'Reference',
@@ -329,6 +346,11 @@
u'Przipisy',
u'Připisy',
],
+ 'th': [
+ u'อ้างอิง',
+ u'เชิงอรรถ',
+ u'หมายเหตุ',
+ ],
'zh': [
u'參考資料',
u'参考资料',
@@ -347,6 +369,9 @@
'be': [u'Зноскі', u'Примечания', u'Reflist', u'Спіс заўваг',
u'Заўвагі'],
'be-x-old': [u'Зноскі'],
+ 'ca': [u'Referències', u'Reflist', u'Listaref', u'Referència',
+ u'Referencies', u'Referències2',
+ u'Amaga', u'Amaga ref', u'Amaga Ref', u'Amaga Ref2', u'Apèndix'],
'da': [u'Reflist'],
'dsb': [u'Referency'],
'en': [u'Reflist', u'Refs', u'FootnotesSmall', u'Reference',
@@ -378,6 +403,7 @@
u'Примечания', u'Список примечаний',
u'Сноска', u'Сноски'],
'szl': [u'Przipisy', u'Připisy'],
+ 'th': [u'รายการอ้างอิง'],
'zh': [u'Reflist', u'RefFoot', u'NoteFoot'],
},
}
@@ -398,6 +424,7 @@
'pl': u'{{Przypisy}}',
'ru': u'{{примечания}}',
'szl': u'{{Przipisy}}',
+ 'th': u'{{รายการอ้างอิง}}',
'zh': u'{{reflist}}',
},
}
@@ -424,13 +451,18 @@
class XmlDumpNoReferencesPageGenerator:
+
"""
Generator which will yield Pages that might lack a references tag.
+
These pages will be retrieved from a local XML dump file
(pages-articles or pages-meta-current).
"""
+
def __init__(self, xmlFilename):
"""
+ Constructor.
+
Arguments:
* xmlFilename - The dump's path, either absolute or relative
"""
@@ -451,7 +483,10 @@
class NoReferencesBot:
+ """References section bot."""
+
def __init__(self, generator, always=False):
+ """Constructor."""
self.generator = generator
self.always = always
self.site = pywikibot.getSite()
@@ -463,19 +498,17 @@
re.IGNORECASE | re.DOTALL)
try:
self.referencesTemplates = referencesTemplates[
- pywikibot.getSite().family.name][pywikibot.getSite().lang]
+ self.site.family.name][self.site.lang]
except KeyError:
self.referencesTemplates = []
try:
self.referencesText = referencesSubstitute[
- pywikibot.getSite().family.name][pywikibot.getSite().lang]
+ self.site.family.name][self.site.lang]
except KeyError:
self.referencesText = u'<references />'
def lacksReferences(self, text, verbose=True):
- """
- Checks whether or not the page is lacking a references tag.
- """
+ """Check whether or not the page is lacking a references tag."""
oldTextCleaned = pywikibot.removeDisabledParts(text)
if self.referencesR.search(oldTextCleaned) or \
self.referencesTagR.search(oldTextCleaned):
@@ -500,14 +533,15 @@
def addReferences(self, oldText):
"""
- Tries to add a references tag into an existing section where it fits
- into. If there is no such section, creates a new section containing
+ Add a references tag into an existing section where it fits into.
+
+ If there is no such section, creates a new section containing
the references tag.
* Returns : The modified pagetext
"""
# Is there an existing section where we can add the references tag?
- for section in pywikibot.translate(self.site, referencesSections):
+ for section in i18n.translate(self.site, referencesSections, fallback=False):
sectionR = re.compile(r'\r?\n=+ *%s *=+ *\r?\n' % section)
index = 0
while index < len(oldText):
@@ -532,7 +566,7 @@
break
# Create a new section for the references tag
- for section in pywikibot.translate(self.site, placeBeforeSections):
+ for section in i18n.translate(self.site, placeBeforeSections, fallback=False):
# Find out where to place the new section
sectionR = re.compile(r'\r?\n(?P<ident>=+) *%s *(?P=ident) *\r?\n'
% section)
@@ -542,7 +576,7 @@
if match:
if pywikibot.isDisabled(oldText, match.start()):
pywikibot.output(
- 'Existing %s section is commented out, won\'t add '
+ 'Existing %s section is commented out, won\'t add '
'the references in front of it.' % section)
index = match.end()
else:
@@ -569,11 +603,11 @@
interwikiPattern = r'\[\[([a-zA-Z\-]+)\s?:([^\[\]\n]*)\]\]\s*'
# won't work with nested templates
# the negative lookahead assures that we'll match the last template
- # occurence in the temp text.
- ### fix me:
- ### {{commons}} or {{commonscat}} are part of Weblinks section
- ### * {{template}} is mostly part of a section
- ### so templatePattern must be fixed
+ # occurrence in the temp text.
+ # FIXME:
+ # {{commons}} or {{commonscat}} are part of Weblinks section
+ # * {{template}} is mostly part of a section
+ # so templatePattern must be fixed
templatePattern = r'\r?\n{{((?!}}).)+?}}\s*'
commentPattern = r'<!--((?!-->).)*?-->\s*'
metadataR = re.compile(r'(\r?\n)?(%s|%s|%s|%s)$'
@@ -598,9 +632,10 @@
newSection = u'\n%s\n' % (self.referencesText)
else:
newSection = u'\n%s %s %s\n%s\n' % (ident,
- pywikibot.translate(
+ i18n.translate(
self.site,
- referencesSections)[0],
+ referencesSections,
+ fallback=False)[0],
ident, self.referencesText)
return oldText[:index] + newSection + oldText[index:]
@@ -660,7 +695,7 @@
pywikibot.output(u"Page %s is a disambig; skipping."
% page.title(asLink=True))
continue
- if pywikibot.getSite().sitename() == 'wikipedia:en' and \
+ if self.site.sitename() == 'wikipedia:en' and \
page.isIpEdit():
pywikibot.output(
u"Page %s is edited by IP. Possible vandalized"
@@ -672,7 +707,9 @@
def main():
- #page generator
+ """Process command line arguments and invoke bot."""
+
+ # page generator
gen = None
# This temporary array is used to read the page title if one single
# page to work on is specified by the arguments.
--
To view, visit https://gerrit.wikimedia.org/r/182442
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ib880d311c97938ca75575a85f120200db387ae9f
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/compat
Gerrit-Branch: master
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: Interwikibot crashes when page is empty
......................................................................
Interwikibot crashes when page is empty
SaveError expects arguments.
The Error class in pywikibot.exceptions requires at least one
additional parameter (the other is defined automatically) since
cde6b11565e2068bf052041dd2ad563658c4faf3 and the interwiki.py
script is calling SaveError without a parameter since
e8c2790c5b8bd38d905ff32c6f5ca9efd7d3721b, which was uploaded about
three years after the change to the Error class (in fact this was
the second change to pywikibot.exceptions only when this happened).
At that time SaveError already subclassed Error so this bug is probably
in it ever since.
(taken from bug report analysed by XZise).
Bug: T85676
Change-Id: Ica77c4e9592085e681d113deb3bb502a8c81bb14
---
M scripts/interwiki.py
1 file changed, 1 insertion(+), 1 deletion(-)
Approvals:
John Vandenberg: Looks good to me, approved
XZise: Looks good to me, but someone else must approve
jenkins-bot: Verified
diff --git a/scripts/interwiki.py b/scripts/interwiki.py
index 342cd5e..55d5667 100755
--- a/scripts/interwiki.py
+++ b/scripts/interwiki.py
@@ -1843,7 +1843,7 @@
raise SaveError(u'Page doesn\'t exist')
if page.isEmpty() and not page.isCategory():
pywikibot.output(u"Not editing %s: page is empty" % page)
- raise SaveError
+ raise SaveError(u'Page is empty.')
# clone original newPages dictionary, so that we can modify it to the
# local page's needs
--
To view, visit https://gerrit.wikimedia.org/r/182425
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ica77c4e9592085e681d113deb3bb502a8c81bb14
Gerrit-PatchSet: 2
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>
jenkins-bot has submitted this change and it was merged.
Change subject: Added a list of urls upload support
......................................................................
Added a list of urls upload support
Test images are now stored in data/images. Use _images_dir as the path variable.
upload_image() is now deprecated, use upload_file(file_url) instead.
Bot's parameter url is deprecated to be string, now it's an iterable.
upload.py does not use urlOK() method anymore. The check is moved to main().
To process_filename() added a check whether there is already a page with the
same title. Docstring for __init__() is extended.
While T85708 is not fixed, try to specify filenames with correct extention.
Bug: T57088
Change-Id: I13d469b3ddcac78b02543efe53868a8672966ec0
---
M scripts/upload.py
M tests/__init__.py
A tests/data/images/1rightarrow.png
R tests/data/images/MP_sounds.png
M tests/data_ingestion_tests.py
M tests/dry_api_tests.py
M tests/upload_tests.py
A tests/uploadbot_tests.py
8 files changed, 232 insertions(+), 74 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/upload.py b/scripts/upload.py
index 91e1976..e3daa44 100755
--- a/scripts/upload.py
+++ b/scripts/upload.py
@@ -23,17 +23,17 @@
'Mi': Mebibytes (1024x1024 B)
The suffixes are case insensitive.
-If any other arguments are given, the first is the URL or filename to upload,
-and the rest is a proposed description to go with the upload. If none of these
-are given, the user is asked for the file or URL to upload. The bot will then
-upload the image to the wiki.
+If any other arguments are given, the first is either URL, filename or directory
+to upload, and the rest is a proposed description to go with the upload. If none
+of these are given, the user is asked for the directory, file or URL to upload.
+The bot will then upload the image to the wiki.
-The script will ask for the location of an image, if not given as a parameter,
+The script will ask for the location of an image(s), if not given as a parameter,
and for a description.
"""
#
# (C) Rob W.W. Hooft, Andre Engels 2003-2004
-# (C) Pywikibot team, 2003-2014
+# (C) Pywikibot team, 2003-2015
#
# Distributed under the terms of the MIT license.
#
@@ -50,6 +50,9 @@
import pywikibot
import pywikibot.data.api
from pywikibot import config
+from pywikibot.tools import (
+ deprecated
+)
if sys.version_info[0] > 2:
from urllib.parse import urlparse
@@ -70,13 +73,44 @@
"""
Constructor.
- @param ignoreWarning: Set this to True if you want to upload even if
- another file would be overwritten or another mistake would be
- risked. You can also set it to an array of warning codes to
- selectively ignore specific warnings.
+ @param url: path to url or local file (deprecated), or list of urls or
+ paths to local files.
+ @type url: string (deprecated) or list
+ @param description: Description of file for its page. If multiple files
+ are uploading the same description is used for every file.
+ @type description: string
+ @param useFilename: Specify title of the file's page. If multiple
+ files are uploading it asks to change the name for second, third,
+ etc. files, otherwise the last file will overwrite the other.
+ @type useFilename: string
+ @param keepFilename: Set to True to keep original names of urls and
+ files, otherwise it will ask to enter a name for each file.
+ @type keepFilename: bool
+ @param verifyDescription: Set to False to not proofread the description.
+ @type verifyDescription: bool
+ @param ignoreWarning: Set this to True to upload even if another file
+ would be overwritten or another mistake would be risked. Set it to
+ an array of warning codes to selectively ignore specific warnings.
+ @type ignoreWarning: bool or list
+ @param targetSite: Set the site to upload to. If target site is not
+ given it's taken from user-config.py.
+ @type targetSite: object
+ @param aborts: List of the warning types to abort upload on. Set to True
+ to abort on any warning.
+ @type aborts: bool or list
+ @param chunk_size: Upload the file in chunks (more overhead, but
+ restartable) specified in bytes. If no value is specified the file
+ will be uploaded as whole.
+ @type chunk_size: integer
+
+ @deprecated: Using upload_image() is deprecated, use upload_file() with
+ file_url param instead
"""
self.url = url
+ if isinstance(self.url, basestring):
+ pywikibot.warning("url as string is deprecated. "
+ "Use an iterable instead.")
self.urlEncoding = urlEncoding
self.description = description
self.useFilename = useFilename
@@ -90,16 +124,21 @@
'commons')
else:
self.targetSite = targetSite or pywikibot.Site()
- self.targetSite.forceLogin()
+ self.targetSite.login()
self.uploadByUrl = uploadByUrl
+ @deprecated()
def urlOK(self):
- """Return True if self.url is an URL or an existing local file."""
+ """Return True if self.url is a URL or an existing local file."""
return "://" in self.url or os.path.exists(self.url)
- def read_file_content(self):
+ def read_file_content(self, file_url=None):
"""Return name of temp file in which remote file is saved."""
- pywikibot.output(u'Reading file %s' % self.url)
+ if not file_url:
+ file_url = self.url
+ pywikibot.warning("file_url is not given. "
+ "Set to self.url by default.")
+ pywikibot.output(u'Reading file %s' % file_url)
resume = False
rlen = 0
_contents = None
@@ -112,7 +151,7 @@
pywikibot.output(u"Resume download...")
uo.addheader('Range', 'bytes=%s-' % rlen)
- infile = uo.open(self.url)
+ infile = uo.open(file_url)
if 'text/html' in infile.info().getheader('Content-Type'):
pywikibot.output(u"Couldn't download the image: "
@@ -150,56 +189,89 @@
pywikibot.log(
u"WARNING: length check of retrieved data not possible.")
handle, tempname = tempfile.mkstemp()
- t = os.fdopen(handle, "wb")
- t.write(_contents)
+ with os.fdopen(handle, "wb") as t:
+ t.write(_contents)
t.close()
return tempname
- def process_filename(self):
- """Return base filename portion of self.url."""
+ def process_filename(self, file_url=None):
+ """Return base filename portion of file_url."""
+ if not file_url:
+ file_url = self.url
+ pywikibot.warning("file_url is not given. "
+ "Set to self.url by default.")
+
# Isolate the pure name
- filename = self.url
- # Filename may be either a local file path or a URL
+ filename = file_url
+ # Filename may be either a URL or a local file path
if "://" in filename:
# extract the path portion of the URL
filename = urlparse(filename).path
filename = os.path.basename(filename)
-
if self.useFilename:
filename = self.useFilename
if not self.keepFilename:
pywikibot.output(
u"The filename on the target wiki will default to: %s"
% filename)
- # FIXME: these 2 belong somewhere else, presumably in family
- forbidden = '/' # to be extended
- allowed_formats = (u'gif', u'jpg', u'jpeg', u'mid', u'midi',
- u'ogg', u'png', u'svg', u'xcf', u'djvu',
- u'ogv', u'oga', u'tif', u'tiff')
- # ask until it's valid
- while True:
- newfn = pywikibot.input(
- u'Enter a better name, or press enter to accept:')
- if newfn == "":
- newfn = filename
- break
- ext = os.path.splitext(newfn)[1].lower().strip('.')
- # are any chars in forbidden also in newfn?
- invalid = set(forbidden) & set(newfn)
- if invalid:
- c = "".join(invalid)
- pywikibot.output(
- 'Invalid character(s): %s. Please try again' % c)
- continue
- if ext not in allowed_formats:
- if not pywikibot.input_yn(
- u"File format is not one of [%s], but %s. Continue?"
- % (u' '.join(allowed_formats), ext),
- default=False, automatic_quit=False):
- continue
- break
- if newfn != '':
+ newfn = pywikibot.input(
+ u'Enter a better name, or press enter to accept:')
+ if newfn != "":
filename = newfn
+ # FIXME: these 2 belong somewhere else, presumably in family
+ # forbidden characters are handled by pywikibot/page.py
+ forbidden = ':*?/\\' # to be extended
+ allowed_formats = (u'gif', u'jpg', u'jpeg', u'mid', u'midi',
+ u'ogg', u'png', u'svg', u'xcf', u'djvu',
+ u'ogv', u'oga', u'tif', u'tiff')
+ # ask until it's valid
+ first_check = True
+ while True:
+ if not first_check:
+ filename = pywikibot.input(u'Enter a better name, '
+ 'or press enter to skip:')
+ if not filename:
+ return None
+ first_check = False
+ ext = os.path.splitext(filename)[1].lower().strip('.')
+ # are any chars in forbidden also in filename?
+ invalid = set(forbidden) & set(filename)
+ if invalid:
+ c = "".join(invalid)
+ pywikibot.output(
+ 'Invalid character(s): %s. Please try again' % c)
+ continue
+ if ext not in allowed_formats:
+ if not pywikibot.input_yn(
+ u"File format is not one of [%s], but %s. Continue?"
+ % (u' '.join(allowed_formats), ext),
+ default=False, automatic_quit=False):
+ continue
+ potential_file_page = pywikibot.FilePage(self.targetSite, filename)
+ if potential_file_page.exists():
+ if potential_file_page.canBeEdited():
+ if pywikibot.input_yn(u"File with name %s already exists. "
+ "Would you like to change the name? "
+ "(Otherwise file will be overwritten.)"
+ % filename, default=True,
+ automatic_quit=False):
+ continue
+ else:
+ break
+ else:
+ pywikibot.output(u"File with name %s already exists and "
+ "cannot be overwritten." % filename)
+ continue
+ else:
+ try:
+ if potential_file_page.fileIsShared():
+ pywikibot.output(u"File with name %s already exists in shared "
+ "repository and cannot be overwritten."
+ % filename)
+ continue
+ except pywikibot.NoPage:
+ break
+
# A proper description for the submission.
# Empty descriptions are not accepted.
pywikibot.output(u'The suggested description is:\n%s'
@@ -247,34 +319,41 @@
else:
return warn_code in self.ignoreWarning
+ @deprecated('UploadRobot.upload_file()')
def upload_image(self, debug=False):
- """Upload the image at self.url to the target wiki.
+ """Upload image."""
+ self.upload_file(self.url, debug)
+
+ def upload_file(self, file_url, debug=False):
+ """Upload the image at file_url to the target wiki.
Return the filename that was used to upload the image.
If the upload fails, ask the user whether to try again or not.
If the user chooses not to retry, return null.
"""
- filename = self.process_filename()
+ filename = self.process_filename(file_url)
+ if not filename:
+ return None
site = self.targetSite
imagepage = pywikibot.FilePage(site, filename) # normalizes filename
imagepage.text = self.description
- pywikibot.output(u'Uploading file to %s via API....' % site)
+ pywikibot.output(u'Uploading file to %s via API...' % site)
try:
apiIgnoreWarnings = False
if self.ignoreWarning is True:
apiIgnoreWarnings = True
if self.uploadByUrl:
- site.upload(imagepage, source_url=self.url,
+ site.upload(imagepage, source_url=file_url,
ignore_warnings=apiIgnoreWarnings)
else:
- if "://" in self.url:
- temp = self.read_file_content()
+ if "://" in file_url:
+ temp = self.read_file_content(file_url)
else:
- temp = self.url
+ temp = file_url
site.upload(imagepage, source_filename=temp,
ignore_warnings=apiIgnoreWarnings,
chunk_size=self.chunk_size)
@@ -307,7 +386,7 @@
else:
# No warning, upload complete.
- pywikibot.output(u"Upload successful.")
+ pywikibot.output(u"Upload of %s successful." % filename)
return filename # data['filename']
def run(self):
@@ -325,14 +404,10 @@
"User '%s' does not have upload rights on site %s."
% (self.targetSite.user(), self.targetSite))
return
-
- while not self.urlOK():
- if not self.url:
- pywikibot.output(u'No input filename given')
- else:
- pywikibot.output(u'Invalid input filename given. Try again.')
- self.url = pywikibot.input(u'File or URL where image is now:')
- return self.upload_image()
+ if isinstance(self.url, basestring):
+ return self.upload_file(self.url)
+ for file_url in self.url:
+ self.upload_file(file_url)
def main(*args):
@@ -403,6 +478,20 @@
url = arg
else:
description.append(arg)
+ while not ("://" in url or os.path.exists(url)):
+ if not url:
+ pywikibot.output(u'No input filename given.')
+ else:
+ pywikibot.output(u'Invalid input filename given. Try again.')
+ url = pywikibot.input(u'URL, file or directory where files are now:')
+ if os.path.isdir(url):
+ file_list = []
+ for directory_info in os.walk(url):
+ for dir_file in directory_info[2]:
+ file_list.append(os.path.join(directory_info[0], dir_file))
+ url = file_list
+ else:
+ url = [url]
description = u' '.join(description)
bot = UploadRobot(url, description=description, useFilename=useFilename,
keepFilename=keepFilename,
diff --git a/tests/__init__.py b/tests/__init__.py
index cd5ef45..df252b7 100644
--- a/tests/__init__.py
+++ b/tests/__init__.py
@@ -51,6 +51,7 @@
_tests_dir = os.path.split(__file__)[0]
_cache_dir = os.path.join(_tests_dir, 'apicache')
_data_dir = os.path.join(_tests_dir, 'data')
+_images_dir = os.path.join(_data_dir, 'images')
# Find the root directory of the checkout
_root_dir = os.path.split(_tests_dir)[0]
diff --git a/tests/data/images/1rightarrow.png b/tests/data/images/1rightarrow.png
new file mode 100644
index 0000000..fbce893
--- /dev/null
+++ b/tests/data/images/1rightarrow.png
Binary files differ
diff --git a/tests/data/MP_sounds.png b/tests/data/images/MP_sounds.png
similarity index 100%
rename from tests/data/MP_sounds.png
rename to tests/data/images/MP_sounds.png
Binary files differ
diff --git a/tests/data_ingestion_tests.py b/tests/data_ingestion_tests.py
index 312bea0..b62a999 100644
--- a/tests/data_ingestion_tests.py
+++ b/tests/data_ingestion_tests.py
@@ -6,6 +6,7 @@
import os
from tests import _data_dir
+from tests import _images_dir
from tests.aspects import unittest, TestCase
from scripts import data_ingestion
@@ -37,7 +38,7 @@
def test_downloadPhoto(self):
"""Test download from http://upload.wikimedia.org/."""
- with open(os.path.join(_data_dir, 'MP_sounds.png'), 'rb') as f:
+ with open(os.path.join(_images_dir, 'MP_sounds.png'), 'rb') as f:
self.assertEqual(f.read(), self.obj.downloadPhoto().read())
def test_findDuplicateImages(self):
diff --git a/tests/dry_api_tests.py b/tests/dry_api_tests.py
index 3b3afe4..75a384a 100644
--- a/tests/dry_api_tests.py
+++ b/tests/dry_api_tests.py
@@ -20,7 +20,7 @@
)
from pywikibot.family import Family
-from tests import _data_dir
+from tests import _images_dir
from tests.utils import DummySiteinfo
from tests.aspects import unittest, TestCase, DefaultDrySiteTestCase
@@ -188,7 +188,7 @@
def test_mime_file_payload(self):
"""Test Request._generate_MIME_part loads binary as binary."""
- local_filename = os.path.join(_data_dir, 'MP_sounds.png')
+ local_filename = os.path.join(_images_dir, 'MP_sounds.png')
with open(local_filename, 'rb') as f:
file_content = f.read()
submsg = Request._generate_MIME_part(
@@ -197,7 +197,7 @@
self.assertEqual(file_content, submsg.get_payload(decode=True))
def test_mime_file_container(self):
- local_filename = os.path.join(_data_dir, 'MP_sounds.png')
+ local_filename = os.path.join(_images_dir, 'MP_sounds.png')
with open(local_filename, 'rb') as f:
file_content = f.read()
body = Request._build_mime_request({}, {
@@ -215,7 +215,7 @@
"""Test Request object prepared to upload."""
req = Request(site=self.get_site(), action="upload",
file='MP_sounds.png', mime=True,
- filename=os.path.join(_data_dir, 'MP_sounds.png'))
+ filename=os.path.join(_images_dir, 'MP_sounds.png'))
self.assertEqual(req.mime, True)
diff --git a/tests/upload_tests.py b/tests/upload_tests.py
index a7f978f..628362b 100644
--- a/tests/upload_tests.py
+++ b/tests/upload_tests.py
@@ -15,7 +15,7 @@
import pywikibot
-from tests import _data_dir
+from tests import _images_dir
from tests.aspects import unittest, TestCase
@@ -31,7 +31,7 @@
def test_png(self):
"""Test uploading a png using Site.upload."""
page = pywikibot.FilePage(self.site, 'MP_sounds-pwb.png')
- local_filename = os.path.join(_data_dir, 'MP_sounds.png')
+ local_filename = os.path.join(_images_dir, 'MP_sounds.png')
self.site.upload(page, source_filename=local_filename,
comment='pywikibot test',
ignore_warnings=True)
@@ -39,7 +39,7 @@
def test_png_chunked(self):
"""Test uploading a png in two chunks using Site.upload."""
page = pywikibot.FilePage(self.site, 'MP_sounds-pwb-chunked.png')
- local_filename = os.path.join(_data_dir, 'MP_sounds.png')
+ local_filename = os.path.join(_images_dir, 'MP_sounds.png')
self.site.upload(page, source_filename=local_filename,
comment='pywikibot test',
ignore_warnings=True, chunk_size=1024)
diff --git a/tests/uploadbot_tests.py b/tests/uploadbot_tests.py
new file mode 100644
index 0000000..a84667c
--- /dev/null
+++ b/tests/uploadbot_tests.py
@@ -0,0 +1,67 @@
+# -*- coding: utf-8 -*-
+"""
+UploadRobot test.
+
+These tests write to the wiki.
+"""
+#
+# (C) Pywikibot team, 2015
+#
+# Distributed under the terms of the MIT license.
+#
+__version__ = '$Id$'
+#
+
+import os
+
+from scripts import upload
+from tests import _images_dir
+from tests.aspects import unittest, TestCase
+
+
+class TestUploadbot(TestCase):
+
+ """Test cases for upload."""
+
+ write = True
+
+ family = 'wikipedia'
+ code = 'test'
+
+ def test_png_list(self):
+ """Test uploading a list of pngs using upload.py."""
+ image_list = []
+ for directory_info in os.walk(_images_dir):
+ for dir_file in directory_info[2]:
+ image_list.append(os.path.join(directory_info[0], dir_file))
+ bot = upload.UploadRobot(url=image_list,
+ description="pywikibot upload.py script test",
+ useFilename=None, keepFilename=True,
+ verifyDescription=True, aborts=set(),
+ ignoreWarning=True, targetSite=self.get_site())
+ bot.run()
+
+ def test_png(self):
+ """Test uploading a png using upload.py."""
+ bot = upload.UploadRobot(url=[os.path.join(_images_dir, "MP_sounds.png")],
+ description="pywikibot upload.py script test",
+ useFilename=None, keepFilename=True,
+ verifyDescription=True, aborts=set(),
+ ignoreWarning=True, targetSite=self.get_site())
+ bot.run()
+
+ def test_png_url(self):
+ """Test uploading a png from url using upload.py."""
+ bot = upload.UploadRobot(url=['https://upload.wikimedia.org/wikipedia/commons/f/fc/MP_sounds.png'],
+ description="pywikibot upload.py script test",
+ useFilename=None, keepFilename=True,
+ verifyDescription=True, aborts=set(),
+ ignoreWarning=True, targetSite=self.get_site())
+ bot.run()
+
+
+if __name__ == '__main__':
+ try:
+ unittest.main()
+ except SystemExit:
+ pass
--
To view, visit https://gerrit.wikimedia.org/r/179149
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I13d469b3ddcac78b02543efe53868a8672966ec0
Gerrit-PatchSet: 13
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Murfel <murnatty(a)gmail.com>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com>
Gerrit-Reviewer: Murfel <murnatty(a)gmail.com>
Gerrit-Reviewer: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: jenkins-bot <>
