[Pywikipedia-l] SVN: [5985] trunk/pywikipedia/checkimages.py
filnik at svn.wikimedia.org
filnik at svn.wikimedia.org
Fri Oct 17 17:54:33 UTC 2008
Revision: 5985
Author: filnik
Date: 2008-10-17 17:54:33 +0000 (Fri, 17 Oct 2008)
Log Message:
-----------
Some rewrite, skip -> in a function
Modified Paths:
--------------
trunk/pywikipedia/checkimages.py
Modified: trunk/pywikipedia/checkimages.py
===================================================================
--- trunk/pywikipedia/checkimages.py 2008-10-16 20:47:02 UTC (rev 5984)
+++ trunk/pywikipedia/checkimages.py 2008-10-17 17:54:33 UTC (rev 5985)
@@ -518,16 +518,19 @@
botolist.append(botnick)
self.botolist = botolist
self.sendemailActive = sendemailActive
+ self.skip_list = list() # Inizialize the skip list used below
self.duplicatesReport = duplicatesReport
image_n = self.site.image_namespace()
- self.image_namespace = "%s:" % image_n # Example: "User_talk:"
+ self.image_namespace = "%s:" % image_n # Example: "Image:"
# Load the licenses only once, so do it once
self.smartdetection = smartdetection
if self.smartdetection:
self.list_licenses = self.load_licenses()
- def setParameters(self, image):
+ def setParameters(self, imageName):
""" Function to set parameters, now only image but maybe it can be used for others in "future" """
- self.image = image
+ self.imageName = imageName
+ # Defing the image's Page Object
+ self.image = wikipedia.ImagePage(self.site, '%s%s' % (self.image_namespace, self.imageName))
def report(self, newtext, image_to_report, notification = None, head = None,
notification2 = None, unver = True, commTalk = None, commImage = None):
""" Function to make the reports easier (or I hope so). """
@@ -590,23 +593,22 @@
def tag_image(self, put = True):
""" Function to add the template in the image and to find out
who's the user that has uploaded the image. """
- # Defing the image's Page Object
- p = wikipedia.ImagePage(self.site, 'Image:%s' % self.image_to_report)
# Get the image's description
+ reportPageObject = wikipedia.ImagePage(self.site, self.image_namespace + self.image_to_report)
try:
- testoa = p.get()
+ reportPageText = reportPageObject.get()
except wikipedia.NoPage:
- wikipedia.output(u'%s has been deleted...' % p.title())
+ wikipedia.output(u'%s has been deleted...' % self.imageName)
# We have a problem! Report and exit!
return False
# You can use this function also to find only the user that
# has upload the image (FixME: Rewrite a bit this part)
if put:
- p.put(testoa + self.newtext, comment = self.commImage, minorEdit = True)
+ reportPageObject.put(reportPageText + self.newtext, comment = self.commImage, minorEdit = True)
# paginetta it's the image page object.
- paginetta = wikipedia.ImagePage(self.site, self.image_namespace + self.image_to_report)
+
try:
- nick = paginetta.getLatestUploader()[0]
+ nick = reportPageObject.getLatestUploader()[0]
except wikipedia.NoPage:
wikipedia.output(u"Seems that %s hasn't the image at all, but there is something in the description..." % self.image_to_report)
repme = "\n*[[:Image:%s]] problems '''with the APIs'''"
@@ -614,9 +616,7 @@
self.report_image(self.image_to_report, self.rep_page, self.com, repme)
return False
luser = wikipedia.url2link(nick, self.site, self.site)
- pagina_discussione = "%s:%s" % (self.site.namespace(3), luser)
- # Defing the talk page (pagina_discussione = talk_page ^__^ )
- talk_page = wikipedia.Page(self.site, pagina_discussione)
+ talk_page = wikipedia.Page(self.site, "%s:%s" % (self.site.namespace(3), luser))
self.talk_page = talk_page
self.luser = luser
return True
@@ -787,34 +787,34 @@
def checkImageOnCommons(self):
""" Checking if the image is on commons """
- wikipedia.output(u'Checking if %s is on commons...' % self.image)
+ wikipedia.output(u'Checking if %s is on commons...' % self.imageName)
commons_site = wikipedia.getSite('commons', 'commons')
- regexOnCommons = r"\n\*\[\[:Image:%s\]\] is also on '''Commons''': \[\[commons:Image:.*?\]\](?: \(same name\)|)$" % self.image
- imagePage = wikipedia.ImagePage(self.site, 'Image:%s' % self.image)
+ regexOnCommons = r"\n\*\[\[:Image:%s\]\] is also on '''Commons''': \[\[commons:Image:.*?\]\](?: \(same name\)|)$" % self.imageName
+ imagePage = wikipedia.ImagePage(self.site, 'Image:%s' % self.imageName)
hash_found = imagePage.getHash()
if hash_found == None:
return False # Problems? Yes! Image deleted, no hash found. Skip the image.
else:
commons_image_with_this_hash = commons_site.getImagesFromAnHash(hash_found)
if commons_image_with_this_hash != []:
- wikipedia.output(u'%s is on commons!' % self.image)
- imagePage = wikipedia.ImagePage(self.site, 'Image:%s' % self.image)
+ wikipedia.output(u'%s is on commons!' % self.imageName)
+ imagePage = wikipedia.ImagePage(self.site, 'Image:%s' % self.imageName)
on_commons_text = imagePage.getImagePageHtml()
if "<div class='sharedUploadNotice'>" in on_commons_text:
wikipedia.output(u"But, the image doesn't exist on your project! Skip...")
# Problems? Yes! We have to skip the check part for that image!
# Because it's on commons but someone has added something on your project.
return False
- elif re.findall(r'\bstemma\b', self.image.lower()) != [] and self.site.lang == 'it':
- wikipedia.output(u'%s has "stemma" inside, means that it\'s ok.' % self.image)
+ elif re.findall(r'\bstemma\b', self.imageName.lower()) != [] and self.site.lang == 'it':
+ wikipedia.output(u'%s has "stemma" inside, means that it\'s ok.' % self.imageName)
return True # Problems? No, it's only not on commons but the image needs a check
else:
# the second usually is a url or something like that. Compare the two in equal way, both url.
- if self.convert_to_url(self.image) == self.convert_to_url(commons_image_with_this_hash[0]):
- repme = "\n*[[:Image:%s]] is also on '''Commons''': [[commons:Image:%s]] (same name)" % (self.image, commons_image_with_this_hash[0])
+ if self.convert_to_url(self.imageName) == self.convert_to_url(commons_image_with_this_hash[0]):
+ repme = "\n*[[:Image:%s]] is also on '''Commons''': [[commons:Image:%s]] (same name)" % (self.imageName, commons_image_with_this_hash[0])
else:
- repme = "\n*[[:Image:%s]] is also on '''Commons''': [[commons:Image:%s]]" % (self.image, commons_image_with_this_hash[0])
- self.report_image(self.image, self.rep_page, self.com, repme, addings = False, regex = regexOnCommons)
+ repme = "\n*[[:Image:%s]] is also on '''Commons''': [[commons:Image:%s]]" % (self.imageName, commons_image_with_this_hash[0])
+ self.report_image(self.imageName, self.rep_page, self.com, repme, addings = False, regex = regexOnCommons)
# Problems? No, return True
return True
else:
@@ -825,7 +825,7 @@
""" Function to check the duplicated images. """
# {{Dupe|Image:Blanche_Montel.jpg}}
# Skip the stub images
- #if 'stub' in self.image.lower() and self.project == 'wikipedia' and self.site.lang == 'it':
+ #if 'stub' in self.imageName.lower() and self.project == 'wikipedia' and self.site.lang == 'it':
# return True # Skip the stub, ok
dupText = wikipedia.translate(self.site, duplicatesText)
dupRegex = wikipedia.translate(self.site, duplicatesRegex)
@@ -833,17 +833,17 @@
dupTalkText = wikipedia.translate(self.site, duplicates_user_talk_text)
dupComment_talk = wikipedia.translate(self.site, duplicates_comment_talk)
dupComment_image = wikipedia.translate(self.site, duplicates_comment_image)
- duplicateRegex = r'\n\*(?:\[\[:Image:%s\]\] has the following duplicates(?: \(\'\'\'forced mode\'\'\'\)|):|\*\[\[:Image:%s\]\])$' % (self.convert_to_url(self.image), self.convert_to_url(self.image))
- imagePage = wikipedia.ImagePage(self.site, 'Image:%s' % self.image)
+ duplicateRegex = r'\n\*(?:\[\[:Image:%s\]\] has the following duplicates(?: \(\'\'\'forced mode\'\'\'\)|):|\*\[\[:Image:%s\]\])$' % (self.convert_to_url(self.imageName), self.convert_to_url(self.imageName))
+ imagePage = wikipedia.ImagePage(self.site, 'Image:%s' % self.imageName)
hash_found = imagePage.getHash()
duplicates = self.site.getImagesFromAnHash(hash_found)
if duplicates == None:
return False # Error, image deleted, no hash found. Skip the image.
if len(duplicates) > 1:
if len(duplicates) == 2:
- wikipedia.output(u'%s has a duplicate! Reporting it...' % self.image)
+ wikipedia.output(u'%s has a duplicate! Reporting it...' % self.imageName)
else:
- wikipedia.output(u'%s has %s duplicates! Reporting them...' % (self.image, len(duplicates) - 1))
+ wikipedia.output(u'%s has %s duplicates! Reporting them...' % (self.imageName, len(duplicates) - 1))
if not dupText == None and not dupRegex == None:
time_image_list = list()
time_list = list()
@@ -919,17 +919,17 @@
commImage = dupComment_image, unver = True)
if self.duplicatesReport or only_report:
if only_report:
- repme = "\n*[[:Image:%s]] has the following duplicates ('''forced mode'''):" % self.convert_to_url(self.image)
+ repme = "\n*[[:Image:%s]] has the following duplicates ('''forced mode'''):" % self.convert_to_url(self.imageName)
else:
- repme = "\n*[[:Image:%s]] has the following duplicates:" % self.convert_to_url(self.image)
+ repme = "\n*[[:Image:%s]] has the following duplicates:" % self.convert_to_url(self.imageName)
for duplicate in duplicates:
- if self.convert_to_url(duplicate) == self.convert_to_url(self.image):
+ if self.convert_to_url(duplicate) == self.convert_to_url(self.imageName):
continue # the image itself, not report also this as duplicate
repme += "\n**[[:Image:%s]]" % self.convert_to_url(duplicate)
- result = self.report_image(self.image, self.rep_page, self.com, repme, addings = False, regex = duplicateRegex)
+ result = self.report_image(self.imageName, self.rep_page, self.com, repme, addings = False, regex = duplicateRegex)
if not result:
return True # If Errors, exit (but continue the check)
- if older_image != self.image:
+ if older_image != self.imageName:
return False # The image is a duplicate, it will be deleted. So skip the check-part, useless
return True # Ok - No problem. Let's continue the checking phase
@@ -1091,13 +1091,13 @@
exit_cicle = True
break
if not seems_ok:
- rep_text_license_fake = "\n*[[:Image:%s]] seems to have a ''fake license'', license detected: {{tl|%s}}." % (self.image, license_found)
- regexFakeLicense = r"\* ?\[\[:Image:%s\]\] seems to have a ''fake license'', license detected: \{\{tl\|%s\}\}\.$" % (self.image, license_found)
- printWithTimeZone(u"%s seems to have a fake license: %s, reporting..." % (self.image, license_found))
- self.report_image(self.image, rep_text = rep_text_license_fake,
+ rep_text_license_fake = "\n*[[:Image:%s]] seems to have a ''fake license'', license detected: {{tl|%s}}." % (self.imageName, license_found)
+ regexFakeLicense = r"\* ?\[\[:Image:%s\]\] seems to have a ''fake license'', license detected: \{\{tl\|%s\}\}\.$" % (self.imageName, license_found)
+ printWithTimeZone(u"%s seems to have a fake license: %s, reporting..." % (self.imageName, license_found))
+ self.report_image(self.imageName, rep_text = rep_text_license_fake,
addings = False, regex = regexFakeLicense)
else:
- printWithTimeZone(u"%s seems ok, license found: %s..." % (self.image, license_found))
+ printWithTimeZone(u"%s seems ok, license found: %s..." % (self.imageName, license_found))
return license_found
def load(self, raw):
@@ -1115,6 +1115,29 @@
list_loaded.append(word)
return list_loaded
+ def skipImages(self, skip_number, limit):
+ # If the images to skip are more the images to check, make them the same number
+ if skip_number == 0:
+ wikipedia.output(u'\t\t>> No images to skip...<<')
+ return False
+ if skip_number > limit: skip_number = limit
+ # Print a starting message only if no images has been skipped
+ if self.skip_list == []:
+ if skip_number == 1:
+ wikipedia.output(u'Skipping the first image:\n')
+ else:
+ wikipedia.output(u'Skipping the first %s images:\n' % skip_number)
+ # If we still have pages to skip:
+ if len(self.skip_list) < skip_number:
+ wikipedia.output(u'Skipping %s...' % self.imageName)
+ self.skip_list.append(self.imageName)
+ if skip_number == 1:
+ wikipedia.output('')
+ return True
+ else:
+ wikipedia.output('') # Print a blank line.
+ return False
+
def checkbot():
""" Main function """
# Command line configurable parameters
@@ -1127,8 +1150,7 @@
normal = False # Check the new images or use another generator?
urlUsed = False # Use the url-related function instead of the new-pages generator
regexGen = False # Use the regex generator
- untagged = False # Use the untagged generator
- skip_list = list() # Inizialize the skip list used below
+ untagged = False # Use the untagged generator
duplicatesActive = False # Use the duplicate option
duplicatesReport = False # Use the duplicate-report option
sendemailActive = False # Use the send-email
@@ -1181,7 +1203,7 @@
firstPageTitle = str(wikipedia.input(u'From witch page do you want to start?'))
elif len(arg) > 6:
firstPageTitle = str(arg[7:])
- generator = wikipedia.getSite().allpages(start=firstPageTitle ,namespace=6)
+ generator = wikipedia.getSite().allpages(start=firstPageTitle, namespace=6)
repeat = False
elif arg.startswith('-page'):
if len(arg) == 5:
@@ -1355,38 +1377,21 @@
mainClass.setParameters(imageName) # Setting the image for the main class
# Skip block
if skip == True:
- # If the images to skip are more the images to check, make them the same number
- if skip_number > limit: skip_number = limit
- # Print a starting message only if no images has been skipped
- if skip_list == []:
- if skip_number == 1:
- wikipedia.output(u'Skipping the first image:\n')
- else:
- wikipedia.output(u'Skipping the first %s images:\n' % skip_number)
- # If we still have pages to skip:
- if len(skip_list) < skip_number:
- wikipedia.output(u'Skipping %s...' % imageName)
- skip_list.append(imageName)
- if skip_number == 1:
- wikipedia.output('')
- skip = False
- continue
- else:
- wikipedia.output('') # Print a blank line.
- skip = False
- elif skip_list == []: # Skip must be false if we are here but
- # the user has set 0 as images to skip
- wikipedia.output(u'\t\t>> No images to skip...<<')
- skip_list.append('skip = Off') # Only to print it once
+ skip = mainClass.skipImages(skip_number, limit)
+ if skip == True:
+ continue
parentesi = False # parentesi are these in italian: { ( ) } []
delete = False
tagged = False
extension = imageName.split('.')[-1] # get the extension from the image's name
# Page => ImagePage
p = wikipedia.ImagePage(site, image.title())
- # Get the text in the image (called g)
+ # Get the text in the image (called imageCheckText)
try:
- g = p.get()
+ # the checkText will be modified in order to make the check phase easier
+ # the imageFullText will be used when the full text is needed without changes
+ imageCheckText = p.get()
+ imageFullText = imageCheckText
except wikipedia.NoPage:
wikipedia.output(u"Skipping %s because it has been deleted." % imageName)
continue
@@ -1396,7 +1401,7 @@
# Delete the fields where the templates cannot be loaded
regex_nowiki = re.compile(r'<nowiki>(.*?)</nowiki>', re.DOTALL)
regex_pre = re.compile(r'<pre>(.*?)</pre>', re.DOTALL)
- g = regex_nowiki.sub('', g); g = regex_pre.sub('', g)
+ imageCheckText = regex_nowiki.sub('', imageCheckText); imageCheckText = regex_pre.sub('', imageCheckText)
# Check on commons if there's already an image with the same name
if commonsActive == True:
response = mainClass.checkImageOnCommons()
@@ -1413,10 +1418,10 @@
# and the regex will be wrong)
if '{{' in i:
regexP = re.compile('\{\{(?:template|)%s ?(?:\||\n|\}|<) ?' % i.split('{{')[1].replace(' ', '[ _]'), re.I)
- result = regexP.findall(g)
+ result = regexP.findall(imageCheckText)
if result != []:
tagged = True
- elif i.lower() in g:
+ elif i.lower() in imageCheckText:
tagged = True
# Deleting the useless template from the description (before adding something
# in the image the original text will be reloaded, don't worry).
@@ -1425,13 +1430,13 @@
for l in hiddentemplate:
if tagged == False:
# why creator? Because on commons there's a template such as {{creator:name}} that.. works
- res = re.findall(r'\{\{(?:[Tt]emplate:|)(?:%s[ \n]*?(?:\n|\||\}|<)|creator:)' % l.lower(), g.lower())
+ res = re.findall(r'\{\{(?:[Tt]emplate:|)(?:%s[ \n]*?(?:\n|\||\}|<)|creator:)' % l.lower(), imageCheckText.lower())
if res != []:
white_template_found += 1
if l != '' and l != ' ': # Check that l is not nothing or a space
# Deleting! (replace the template with nothing)
regex_white_template = re.compile(r'\{\{(?:template:|)(?:%s|creator)' % l, re.IGNORECASE)
- g = regex_white_template.sub(r'', g)
+ imageCheckText = regex_white_template.sub(r'', imageCheckText)
hiddenTemplateFound = True
if white_template_found == 1:
wikipedia.output(u'A white template found, skipping the template...')
@@ -1440,7 +1445,7 @@
else:
wikipedia.output(u'White templates found: %s; skipping those templates...' % white_template_found)
for a_word in something: # something is the array with {{, MIT License and so on.
- if a_word in g:
+ if a_word in imageCheckText:
# There's a template, probably a license (or I hope so)
parentesi = True
# Is the extension allowed? (is it an image or f.e. a .xls file?)
@@ -1474,7 +1479,7 @@
wikipedia.setAction(summary)
for k in find_list:
if find_tipe.lower() == 'findonly':
- if k.lower() == g.lower():
+ if k.lower() == imageCheckText.lower():
some_problem = True
text_used = text
head_used = head_2
@@ -1484,7 +1489,7 @@
mex_used = mexCatched
break
elif find_tipe.lower() == 'find':
- if k.lower() in g.lower():
+ if k.lower() in imageCheckText.lower():
some_problem = True
text_used = text
head_used = head_2
@@ -1503,7 +1508,7 @@
printWithTimeZone(u'%s is already tagged...' % imageName)
continue
if some_problem == True:
- if mex_used in g:
+ if mex_used in imageCheckText:
wikipedia.output(u'Image already fixed. Skip.')
continue
wikipedia.output(u"The image description for %s contains %s..." % (imageName, name_used))
@@ -1524,7 +1529,7 @@
seems_ok = False
license_found = None
if smartdetection:
- license_found = mainClass.smartDetection(g)
+ license_found = mainClass.smartDetection(imageCheckText)
else:
printWithTimeZone(u"%s seems ok..." % imageName)
# It works also without this... but i want only to be sure ^^
@@ -1540,7 +1545,7 @@
mainClass.report(canctext, imageName, notification, head)
delete = False
continue
- elif g in nothing:
+ elif imageCheckText in nothing:
wikipedia.output(u"The image description for %s does not contain a license template!" % imageName)
if hiddenTemplateFound and HiddenTN != None and HiddenTN != '' and HiddenTN != ' ':
notification = HiddenTN % imageName
More information about the Pywikipedia-l
mailing list