jenkins-bot has submitted this change and it was merged.
Change subject: [BUGFIX] upload is to be imported, pep8 changes
......................................................................
[BUGFIX] upload is to be imported, pep8 changes
Change-Id: I31bd49aedf2f1d3aa7a2b94f243dc4c81a2ee7cd
---
M scripts/data_ingestion.py
1 file changed, 48 insertions(+), 35 deletions(-)
Approvals:
Merlijn van Deen: Looks good to me, approved
jenkins-bot: Verified
diff --git a/scripts/data_ingestion.py b/scripts/data_ingestion.py
index f8bc376..38d5246 100755
--- a/scripts/data_ingestion.py
+++ b/scripts/data_ingestion.py
@@ -11,7 +11,6 @@
__version__ = '$Id$'
#
-import pywikibot
import posixpath
import urlparse
import urllib
@@ -19,57 +18,66 @@
import base64
import StringIO
+import pywikibot
+import upload
+
class Photo(object):
- '''
+ """
Represents a Photo (or other file), with metadata, to upload to Commons.
- The constructor takes two parameters: URL (string) and metadata (dict with str:str key:value pairs)
- that can be referred to from the title & template generation.
+ The constructor takes two parameters: URL (string) and metadata (dict with
+ str:str key:value pairs) that can be referred to from the title & template
+ generation.
- '''
+ """
+
def __init__(self, URL, metadata):
self.URL = URL
self.metadata = metadata
self.metadata["_url"] = URL
- self.metadata["_filename"] = filename = posixpath.split(urlparse.urlparse(URL)[2])[1]
+ self.metadata["_filename"] = filename = posixpath.split(
+ urlparse.urlparse(URL)[2])[1]
self.metadata["_ext"] = ext = filename.split(".")[-1]
if ext == filename:
self.metadata["_ext"] = ext = None
self.contents = None
def downloadPhoto(self):
- '''
+ """
Download the photo and store it in a StringIO.StringIO object.
TODO: Add exception handling
- '''
+ """
if not self.contents:
imageFile = urllib.urlopen(self.URL).read()
self.contents = StringIO.StringIO(imageFile)
return self.contents
- def findDuplicateImages(self, site=pywikibot.getSite(u'commons', u'commons')):
- '''
- Takes the photo, calculates the SHA1 hash and asks the mediawiki api for a list of duplicates.
+ def findDuplicateImages(self,
+ site=pywikibot.getSite(u'commons', u'commons')):
+ """
+ Takes the photo, calculates the SHA1 hash and asks the mediawiki api
+ for a list of duplicates.
TODO: Add exception handling, fix site thing
- '''
+ """
hashObject = hashlib.sha1()
hashObject.update(self.downloadPhoto().getvalue())
return site.getFilesFromAnHash(base64.b16encode(hashObject.digest()))
def getTitle(self, fmt):
"""
- Given a format string with %(name)s entries, returns the string formatted with metadata
+ Given a format string with %(name)s entries, returns the string
+ formatted with metadata
"""
return fmt % self.metadata
def getDescription(self, template, extraparams={}):
- '''
+ """
Generate a description for a file
- '''
+ """
params = {}
params.update(self.metadata)
@@ -78,7 +86,8 @@
for key in sorted(params.keys()):
value = params[key]
if not key.startswith("_"):
- description = description + (u'|%s=%s' % (key, self._safeTemplateValue(value))) + "\n"
+ description = description + (
+ u'|%s=%s' % (key, self._safeTemplateValue(value))) + "\n"
description = description + u'}}'
return description
@@ -96,7 +105,8 @@
class DataIngestionBot:
- def __init__(self, reader, titlefmt, pagefmt, site=pywikibot.getSite(u'commons', u'commons')):
+ def __init__(self, reader, titlefmt, pagefmt,
+ site=pywikibot.getSite(u'commons', u'commons')):
self.reader = reader
self.titlefmt = titlefmt
self.pagefmt = pagefmt
@@ -105,7 +115,7 @@
def _doUpload(self, photo):
duplicates = photo.findDuplicateImages(self.site)
if duplicates:
- pywikibot.output(u"Skipping duplicate of %r" % (duplicates, ))
+ pywikibot.output(u"Skipping duplicate of %r" % duplicates)
return duplicates[0]
title = photo.getTitle(self.titlefmt)
@@ -132,22 +142,25 @@
if __name__ == "__main__":
reader = CSVReader(open('tests/data/csv_ingestion.csv'), 'url')
- bot = DataIngestionBot(reader, "%(name)s - %(set)s.%(_ext)s", ":user:valhallasw/test_template", pywikibot.getSite('test', 'test'))
+ bot = DataIngestionBot(
+ reader,
+ "%(name)s - %(set)s.%(_ext)s", ":user:valhallasw/test_template",
+ pywikibot.getSite('test', 'test'))
bot.run()
-"""
+'''
class DataIngestionBot:
def __init__(self, configurationPage):
- '''
+ """
- '''
+ """
self.site = configurationPage.site()
self.configuration = self.parseConfigurationPage(configurationPage)
def parseConfigurationPage(self, configurationPage):
- '''
+ """
Expects a pywikibot.page object "configurationPage" which contains the configuration
- '''
+ """
configuration = {}
# Set a bunch of defaults
configuration['csvDialect']=u'excel'
@@ -169,30 +182,30 @@
def downloadPhoto(self, photoUrl = ''):
- '''
+ """
Download the photo and store it in a StrinIO.StringIO object.
TODO: Add exception handling
- '''
+ """
imageFile=urllib.urlopen(photoUrl).read()
return StringIO.StringIO(imageFile)
def findDuplicateImages(self, photo = None, site = pywikibot.getSite(u'commons', u'commons')):
- '''
+ """
Takes the photo, calculates the SHA1 hash and asks the mediawiki api for a list of duplicates.
TODO: Add exception handling, fix site thing
- '''
+ """
hashObject = hashlib.sha1()
hashObject.update(photo.getvalue())
return site.getFilesFromAnHash(base64.b16encode(hashObject.digest()))
def getTitle(self, metadata):
- '''
+ """
Build a title.
Have titleFormat to indicate how the title would look.
We need to be able to strip off stuff if it's too long. configuration.get('maxTitleLength')
- '''
+ """
#FIXME: Make this configurable.
title = self.configuration.get('titleFormat') % metadata
@@ -208,9 +221,9 @@
return flickrripper.cleanUpTitle(title)
def cleanDate(self, field):
- '''
+ """
A function to do date clean up.
- '''
+ """
# Empty, make it really empty
if field==u'-':
return u''
@@ -259,9 +272,9 @@
self.processFile(metadata)
def run(self):
- '''
+ """
Do crap
- '''
+ """
if not self.configuration.get('sourceFormat'):
pywikibot.output(u'The field "sourceFormat" is not set')
return False
@@ -292,4 +305,4 @@
main(sys.argv[1:])
finally:
print "All done!"
-"""
+'''
--
To view, visit https://gerrit.wikimedia.org/r/100132
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I31bd49aedf2f1d3aa7a2b94f243dc4c81a2ee7cd
Gerrit-PatchSet: 3
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Xqt <info(a)gno.de>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: jenkins-bot
jenkins-bot has submitted this change and it was merged.
Change subject: Bug 58053 - py3k-ify pwb.py, check py version
......................................................................
Bug 58053 - py3k-ify pwb.py, check py version
- Use builtins instead of __builtin__
- Check for PY3 environment variable before complaining
- check for the right py version including py3k
Change-Id: If9090c10cf00154b275053fc2aa60f33ba85181b
---
M pwb.py
1 file changed, 21 insertions(+), 6 deletions(-)
Approvals:
Xqt: Looks good to me, but someone else must approve
Merlijn van Deen: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pwb.py b/pwb.py
index ae9190a..86c9b9e 100644
--- a/pwb.py
+++ b/pwb.py
@@ -48,7 +48,10 @@
main_mod = imp.new_module('__main__')
sys.modules['__main__'] = main_mod
main_mod.__file__ = filename
- main_mod.__builtins__ = sys.modules['__builtin__']
+ if sys.version_info[0] == 2:
+ main_mod.__builtins__ = sys.modules['__builtin__']
+ else:
+ main_mod.builtins = sys.modules['builtins']
# Set sys.argv and the first path element properly.
old_argv = sys.argv
@@ -73,11 +76,23 @@
#### end of snippet
-if sys.version_info[0] != 2:
- raise RuntimeError("ERROR: Pywikipediabot only runs under Python 2")
-if sys.version_info[1] < 6:
- raise RuntimeError("ERROR: Pywikipediabot only runs under Python 2.6 "
- "or higher")
+if not os.environ.get("PY3", False):
+ if sys.version_info[0] != 2:
+ raise RuntimeError("ERROR: Pywikibot only runs under Python 2")
+ if sys.version_info[1] < 6:
+ raise RuntimeError("ERROR: Pywikibot only runs under Python 2.6 "
+ "or higher")
+else:
+ if sys.version_info[0] not in (2, 3):
+ raise RuntimeError("ERROR: Pywikipediabot only runs under Python 2 "
+ "or Python 3")
+ version = tuple(sys.version_info)[:3]
+ if version < (2, 6):
+ raise RuntimeError("ERROR: Pywikibot only runs under Python 2.6 "
+ "or higher")
+ if version >= (3, ) and version < (3, 3):
+ raise RuntimeError("ERROR: Pywikibot only runs under Python 3.3 "
+ "or higher")
rewrite_path = os.path.dirname(sys.argv[0])
if not os.path.isabs(rewrite_path):
--
To view, visit https://gerrit.wikimedia.org/r/98385
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: If9090c10cf00154b275053fc2aa60f33ba85181b
Gerrit-PatchSet: 8
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot
jenkins-bot has submitted this change and it was merged.
Change subject: BUGFIX: correct import order
......................................................................
BUGFIX: correct import order
from urllib.parse import quote_from_bytes, unquote_to_bytes is the python 3
version, while from urllib import quote as quote_from_bytes, unquote as
unquote_to_bytes is the python 2 version -- not the other way around.
Change-Id: Id3b61995e9685d87255fbce374a4118c372ddc2b
---
M pywikibot/page.py
1 file changed, 2 insertions(+), 2 deletions(-)
Approvals:
Xqt: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/page.py b/pywikibot/page.py
index f3807dc..a4a946c 100644
--- a/pywikibot/page.py
+++ b/pywikibot/page.py
@@ -20,11 +20,11 @@
try:
import htmlentitydefs
- from urllib.parse import quote_from_bytes, unquote_to_bytes
+ from urllib import quote as quote_from_bytes, unquote as unquote_to_bytes
except ImportError:
unicode = str
from html import entities as htmlentitydefs
- from urllib import quote as quote_from_bytes, unquote as unquote_to_bytes
+ from urllib.parse import quote_from_bytes, unquote_to_bytes
import logging
import re
--
To view, visit https://gerrit.wikimedia.org/r/100148
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Id3b61995e9685d87255fbce374a4118c372ddc2b
Gerrit-PatchSet: 1
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: jenkins-bot