Pywikipedia-svn October 2009

pywikipedia-svn@lists.wikimedia.org

10 participants
237 discussions

SVN: [7496] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 18 Oct '09

18 Oct '09

Revision: 7496 Author: alexsh Date: 2009-10-18 17:40:03 +0000 (Sun, 18 Oct 2009) Log Message: ----------- ImagePage().getFileMd5Sum(): revert Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-18 17:24:37 UTC (rev 7495) +++ trunk/pywikipedia/wikipedia.py 2009-10-18 17:40:03 UTC (rev 7496) @@ -3536,10 +3536,9 @@ # (see bug #1795683). def getFileMd5Sum(self): """Return image file's MD5 checksum.""" - - f = self.site().getUrl(self.fileUrl(), no_hostname=True) - md5Checksum = md5(f).hexdigest() - return md5Checksum + uo = MyURLopener() + f = uo.open(self.fileUrl()) + return md5(f.read()).hexdigest() def getFileVersionHistory(self): """Return the image file's version history.

1 0

SVN: [7495] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 18 Oct '09

18 Oct '09

Revision: 7495 Author: alexsh Date: 2009-10-18 17:24:37 +0000 (Sun, 18 Oct 2009) Log Message: ----------- ImagePage(): handle NotImplementedError Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-18 16:42:51 UTC (rev 7494) +++ trunk/pywikipedia/wikipedia.py 2009-10-18 17:24:37 UTC (rev 7495) @@ -737,8 +737,8 @@ 'prop': ['revisions','info'], 'rvprop': ['content','ids','flags','timestamp','user','comment','size'], 'rvlimit': 1, - 'inprop': 'protection',#|talkid|subjectid', - 'intoken': 'edit', + 'inprop': ['protection','talkid','subjectid','url','readable'], + #'intoken': 'edit', } if oldid: params['rvstartid'] = oldid @@ -776,13 +776,13 @@ if change_edit_time: self._editTime = parsetime2stamp(pageInfo['revisions'][0]['timestamp']) - self._startTime = parsetime2stamp(pageInfo["starttimestamp"]) + if "starttimestamp" in pageInfo: + self._startTime = parsetime2stamp(pageInfo["starttimestamp"]) self._isWatched = False #cannot handle in API in my research for now. - pagetext = pageInfo['revisions'][0]['*'] - pagetext = unescape(pagetext) + pagetext = unescape(pageInfo['revisions'][0]['*']) pagetext = pagetext.rstrip() if self.site().lang == 'eo': pagetext = decodeEsperantoX(pagetext) @@ -3432,7 +3432,7 @@ if self.namespace() != 6: raise ValueError(u'BUG: %s is not in the image namespace!' % title) self._imagePageHtml = None - self._local = True + self._local = None self._latestInfo = {} self._infoLoaded = False @@ -3456,12 +3456,20 @@ 'iiprop': ['timestamp', 'user', 'comment', 'url', 'size', 'dimensions', 'sha1', 'mime', 'metadata', 'archivename', 'bitdepth'], 'iilimit': limit, } - data = query.GetData(params, self.site()) + try: + data = query.GetData(params, self.site()) + except NotImplementedError: + output("API not work, loading page HTML.") + self.getImagePageHtml() + return + if 'error' in data: raise RuntimeError("%s" %data['error']) count = 0 if data['query']['pages'].values()[0]["imagerepository"] == "shared": self._local = False + else: + self._local = True infos = [] while True: @@ -3499,17 +3507,24 @@ if not self._infoLoaded: self._loadInfo() - return self.latestInfo['url'] - #urlR = re.compile(r'<div class="fullImageLink" id="file">.*?<a href="(?P<url>[^ ]+?)"(?! class="image")|<span class="dangerousLink"><a href="(?P<url2>.+?)"', re.DOTALL) - #m = urlR.search(self.getImagePageHtml()) + if self._infoLoaded: + return self.latestInfo['url'] + + urlR = re.compile(r'<div class="fullImageLink" id="file">.*?<a href="(?P<url>[^ ]+?)"(?! class="image")|<span class="dangerousLink"><a href="(?P<url2>.+?)"', re.DOTALL) + m = urlR.search(self.getImagePageHtml()) - # url = m.group('url') or m.group('url2') + url = m.group('url') or m.group('url2') + return url def fileIsOnCommons(self): """Return True if the image is stored on Wikimedia Commons""" if not self._infoLoaded: self._loadInfo() - return not self._local + + if self._infoLoaded: + return not self._local + + return self.fileUrl().startswith(u'http://upload.wikimedia.org/wikipedia/commons/') def fileIsShared(self): """Return True if image is stored on Wikitravel shared repository.""" @@ -3575,7 +3590,7 @@ self._loadInfo() try: return self._latestInfo['sha1'] - except (KeyError, IndexError): + except (KeyError, IndexError, TypeError): try: self.get() except NoPage: @@ -3598,6 +3613,38 @@ return u'{| border="1"\n! date/time || username || resolution || size || edit summary\n|----\n' + u'\n|----\n'.join(lines) + '\n|}' def usingPages(self): + try: + if config.use_api and self.site().versionnumber() >= 11: + x = self.site().api_address() + del x + else: + raise NotImplementedError + except NotImplementedError: + for a in self._usingPagesOld(): + yield a + return + params = { + 'action': 'query', + 'list': 'imageusage', + 'iutitle': self.title(), + 'iulimit': config.special_page_limit, + #'': '', + } + + while True: + data = query.GetData(params, self.site()) + if 'error' in data: + raise RuntimeError("%s" % data['error']) + + for iu in data['query']["imageusage"]: + yield Page(self.site(), iu['title'], defaultNamespace=iu['ns']) + + if 'query-continue' in data: + params['iucontinue'] = data['query-continue']['imageusage']['iucontinue'] + else: + break + + def _usingPagesOld(self): """Yield Pages on which the image is displayed.""" titleList = re.search('(?s)<h2 id="filelinks">.+?', self.getImagePageHtml()).group()

1 0

SVN: [7494] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 18 Oct '09

18 Oct '09

Revision: 7494 Author: alexsh Date: 2009-10-18 16:42:51 +0000 (Sun, 18 Oct 2009) Log Message: ----------- ImagePage():load page Info in one time load Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-18 06:23:41 UTC (rev 7493) +++ trunk/pywikipedia/wikipedia.py 2009-10-18 16:42:51 UTC (rev 7494) @@ -3406,7 +3406,7 @@ raise NoPage(u'API Error, nothing found in the APIs') # We don't know the page's id, if any other better idea please change it - return data[data.keys()[0]][u'revisions'] + return data.values()[0][u'revisions'] class ImagePage(Page): """A subclass of Page representing an image descriptor wiki page. @@ -3432,6 +3432,9 @@ if self.namespace() != 6: raise ValueError(u'BUG: %s is not in the image namespace!' % title) self._imagePageHtml = None + self._local = True + self._latestInfo = {} + self._infoLoaded = False def getImagePageHtml(self): """ @@ -3445,6 +3448,42 @@ self._imagePageHtml = self.site().getUrl(path) return self._imagePageHtml + def _loadInfo(self, limit = 1): + params = { + 'action': 'query', + 'prop': 'imageinfo', + 'titles': self.title(), + 'iiprop': ['timestamp', 'user', 'comment', 'url', 'size', 'dimensions', 'sha1', 'mime', 'metadata', 'archivename', 'bitdepth'], + 'iilimit': limit, + } + data = query.GetData(params, self.site()) + if 'error' in data: + raise RuntimeError("%s" %data['error']) + count = 0 + if data['query']['pages'].values()[0]["imagerepository"] == "shared": + self._local = False + infos = [] + + while True: + for info in data['query']['pages'].values()[0]['imageinfo']: + count += 1 + if count == 1 and 'iistart' not in params: + # count 1 and no iicontinue mean first image revision is latest. + self.latestInfo = info + infos.append(info) + if count >= limit: + break + + + if count < limit and 'query-continue' in data: + params['iistart'] = data['query-continue']['imageinfo']['iistart'] + else: + break + + self._infoLoaded = True + if limit > 1: + return infos + def fileUrl(self): """Return the URL for the image described on this page.""" # There are three types of image pages: @@ -3457,27 +3496,20 @@ # is a full image link for .ogg and .mid files. #*********************** #change to API query: action=query&titles=File:wiki.jpg&prop=imageinfo&iiprop=url - params = { - 'action' :'query', - 'prop' :'imageinfo', - 'titles' :self.title(), - 'iiprop' :'url', - } - imagedata = query.GetData(params, self.site(), encodeTitle = False) - try: - url = imagedata['query']['pages'].values()[0]['imageinfo'][0]['url'] - return url + if not self._infoLoaded: + self._loadInfo() + + return self.latestInfo['url'] #urlR = re.compile(r'<div class="fullImageLink" id="file">.*?<a href="(?P<url>[^ ]+?)"(?! class="image")|<span class="dangerousLink"><a href="(?P<url2>.+?)"', re.DOTALL) #m = urlR.search(self.getImagePageHtml()) # url = m.group('url') or m.group('url2') - except KeyError: - raise NoPage(u'Image file URL for %s not found.' % self.aslink(forceInterwiki = True) ) - return url def fileIsOnCommons(self): """Return True if the image is stored on Wikimedia Commons""" - return self.fileUrl().startswith(u'http://upload.wikimedia.org/wikipedia/commons/') + if not self._infoLoaded: + self._loadInfo() + return not self._local def fileIsShared(self): """Return True if image is stored on Wikitravel shared repository.""" @@ -3489,9 +3521,9 @@ # (see bug #1795683). def getFileMd5Sum(self): """Return image file's MD5 checksum.""" - uo = MyURLopener() - f = uo.open(self.fileUrl()) - md5Checksum = md5(f.read()).hexdigest() + + f = self.site().getUrl(self.fileUrl(), no_hostname=True) + md5Checksum = md5(f).hexdigest() return md5Checksum def getFileVersionHistory(self): @@ -3526,18 +3558,11 @@ def getLatestUploader(self): """ Function that uses the APIs to detect the latest uploader of the image """ - params = { - 'action' :'query', - 'prop' :'imageinfo', - 'titles' :self.title(), - } - data = query.GetData(params, self.site(), encodeTitle = False) + if not self._infoLoaded: + self._loadInfo() try: # We don't know the page's id, if any other better idea please change it - pageid = data['query']['pages'].keys()[0] - nick = data['query']['pages'][pageid][u'imageinfo'][0][u'user'] - timestamp = data['query']['pages'][pageid][u'imageinfo'][0][u'timestamp'] - return [nick, timestamp] + return [self.latestInfo['user'], self.latestInfo['timestamp']] except KeyError: raise NoPage(u'API Error, nothing found in the APIs') @@ -3546,17 +3571,10 @@ Files are the same or not. """ if self.exists(): - params = { - 'action' :'query', - 'titles' :self.title(), - 'prop' :'imageinfo', - 'iiprop' :'sha1', - } - # First of all we need the Hash that identify an image - data = query.GetData(params, self.site(), encodeTitle = False) - pageid = data['query']['pages'].keys()[0] + if not self._infoLoaded: + self._loadInfo() try: - hash_found = data['query']['pages'][pageid][u'imageinfo'][0][u'sha1'] + return self._latestInfo['sha1'] except (KeyError, IndexError): try: self.get() @@ -3568,8 +3586,6 @@ return None else: raise NoHash('No Hash found in the APIs! Maybe the regex to catch it is wrong or someone has changed the APIs structure.') - else: - return hash_found else: output(u'File deleted before getting the Hash. Skipping...') return None

1 0

SVN: [7493] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 17 Oct '09

17 Oct '09

Revision: 7493 Author: alexsh Date: 2009-10-18 06:23:41 +0000 (Sun, 18 Oct 2009) Log Message: ----------- code cleanup Modified Paths: -------------- trunk/pywikipedia/checkimages.py trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2009-10-17 18:34:18 UTC (rev 7492) +++ trunk/pywikipedia/checkimages.py 2009-10-18 06:23:41 UTC (rev 7493) @@ -506,8 +506,6 @@ self.logFulNumber = logFulNumber - self.settings = wikipedia.translate(self.site, page_with_settings) - self.rep_page = wikipedia.translate(self.site, report_page) self.rep_text = wikipedia.translate(self.site, report_text) @@ -1050,11 +1048,12 @@ def takesettings(self): """ Function to take the settings from the wiki. """ + settingsPage = wikipedia.translate(self.site, page_with_settings) try: - if not self.settings: + if not settingsPage: self.settingsData = None else: - wikiPage = wikipedia.Page(self.site, self.settings) + wikiPage = wikipedia.Page(self.site, settingsPage) self.settingsData = list() try: testo = wikiPage.get() @@ -1171,7 +1170,7 @@ result = self.miniTemplateCheck(template) if result: break - if self.license_found == None: + if not self.license_found: for template in self.licenses_found: try: template.pageAPInfo() @@ -1226,10 +1225,10 @@ #else: break - if self.licenses_found != []: + if self.licenses_found: self.templateInList() - if self.license_found == None and self.allLicenses != list(): + if not self.license_found and self.allLicenses: # If only iterlist = self.AllLicenses if I remove something # from iterlist it will be remove from self.AllLicenses too iterlist = list(self.allLicenses) @@ -1242,42 +1241,42 @@ except wikipedia.NoPage: self.allLicenses.remove(template) - if self.allLicenses != list(): + if self.allLicenses: self.license_found = self.allLicenses[0].title() self.some_problem = False # If it has "some_problem" it must check # the additional settings. # if self.settingsData, use addictional settings - if self.settingsData != None: + if self.settingsData: self.findAdditionalProblems() - if self.some_problem == False: - if not self.seems_ok and self.license_found != None: - rep_text_license_fake = u"\n*[[:File:%s]] seems to have " % self.imageName + \ - "a ''fake license'', license detected: <nowiki>%s</nowiki>" % self.license_found - regexFakeLicense = r"\* ?\[\[:File:%s\]\] seems to have " % (re.escape(self.imageName)) + \ - "a ''fake license'', license detected: <nowiki>%s</nowiki>$" % (re.escape(self.license_found)) - printWithTimeZone(u"%s seems to have a fake license: %s, reporting..." % (self.imageName, self.license_found)) - self.report_image(self.imageName, rep_text = rep_text_license_fake, - addings = False, regex = regexFakeLicense) - elif self.license_found != None: - printWithTimeZone(u"%s seems ok, license found: %s..." % (self.imageName, self.license_found)) - else: + if self.some_problem: if self.mex_used in self.imageCheckText: wikipedia.output(u'File already fixed. Skip.') else: wikipedia.output(u"The file's description for %s contains %s..." % (self.imageName, self.name_used)) if self.mex_used.lower() == 'default': self.mex_used = self.unvertext - if self.imagestatus_used == False: - reported = self.report_image(self.imageName) - else: + if self.imagestatus_used: reported = True - if reported == True: + else: + reported = self.report_image(self.imageName) + if reported: #if self.imagestatus_used == True: self.report(self.mex_used, self.imageName, self.text_used, u"\n%s\n" % self.head_used, None, self.imagestatus_used, self.summary_used) else: wikipedia.output(u"Skipping the file...") self.some_problem = False + else: + if not self.seems_ok and self.license_found: + rep_text_license_fake = u"\n*[[:File:%s]] seems to have " % self.imageName + \ + "a ''fake license'', license detected: <nowiki>%s</nowiki>" % self.license_found + regexFakeLicense = r"\* ?\[\[:File:%s\]\] seems to have " % (re.escape(self.imageName)) + \ + "a ''fake license'', license detected: <nowiki>%s</nowiki>$" % (re.escape(self.license_found)) + printWithTimeZone(u"%s seems to have a fake license: %s, reporting..." % (self.imageName, self.license_found)) + self.report_image(self.imageName, rep_text = rep_text_license_fake, + addings = False, regex = regexFakeLicense) + elif self.license_found: + printWithTimeZone(u"%s seems ok, license found: %s..." % (self.imageName, self.license_found)) return (self.license_found, self.whiteTemplatesFound) def load(self, raw): @@ -1303,7 +1302,7 @@ return False if skip_number > limit: skip_number = limit # Print a starting message only if no images has been skipped - if self.skip_list == []: + if not self.skip_list: if skip_number == 1: wikipedia.output(u'Skipping the first file:\n') else: @@ -1386,19 +1385,19 @@ def isTagged(self): """ Understand if a file is already tagged or not. """ - TextFind = wikipedia.translate(self.site, txt_find) # Is the image already tagged? If yes, no need to double-check, skip - for i in TextFind: + for i in wikipedia.translate(self.site, txt_find): # If there are {{ use regex, otherwise no (if there's not the {{ may not be a template # and the regex will be wrong) if '{{' in i: regexP = re.compile(r'\{\{(?:template|)%s ?(?:\||\n|\}|<) ?' % i.split('{{')[1].replace(u' ', u'[ _]'), re.I) result = regexP.findall(self.imageCheckText) - if result != []: + if result: return True elif i.lower() in self.imageCheckText: return True - return False # Nothing Found? Ok: False + + return False # Nothing Found def findAdditionalProblems(self): # In every tupla there's a setting configuration @@ -1418,8 +1417,7 @@ break summary = tupla[5] head_2 = tupla[6] - text = tupla[7] - text = text % self.imageName + text = tupla[7] % self.imageName mexCatched = tupla[8] for k in find_list: if find_tipe.lower() == 'findonly': @@ -1491,8 +1489,7 @@ self.imageCheckText = regex_nowiki.sub('', self.imageCheckText); self.imageCheckText = regex_pre.sub('', self.imageCheckText) # Deleting the useless template from the description (before adding something # in the image the original text will be reloaded, don't worry). - self.tagged = self.isTagged() - if self.tagged == True: + if self.isTagged(): # Tagged? Yes, skip. printWithTimeZone(u'%s is already tagged...' % self.imageName) return True @@ -1514,7 +1511,7 @@ # It works also without this... but i want only to be sure ^^ brackets = False return True - elif delete == True: + elif delete: wikipedia.output(u"%s is not a file!" % self.imageName) # Modify summary text wikipedia.setAction(dels) Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-17 18:34:18 UTC (rev 7492) +++ trunk/pywikipedia/wikipedia.py 2009-10-18 06:23:41 UTC (rev 7493) @@ -3467,25 +3467,22 @@ try: url = imagedata['query']['pages'].values()[0]['imageinfo'][0]['url'] return url -# urlR = re.compile(r'<div class="fullImageLink" id="file">.*?<a href="(?P<url>[^ ]+?)"(?! class="image")|<span class="dangerousLink"><a href="(?P<url2>.+?)"', re.DOTALL) -# m = urlR.search(self.getImagePageHtml()) + #urlR = re.compile(r'<div class="fullImageLink" id="file">.*?<a href="(?P<url>[^ ]+?)"(?! class="image")|<span class="dangerousLink"><a href="(?P<url2>.+?)"', re.DOTALL) + #m = urlR.search(self.getImagePageHtml()) -# url = m.group('url') or m.group('url2') + # url = m.group('url') or m.group('url2') except KeyError: - raise NoPage(u'Image file URL for %s not found.' - % self.aslink(forceInterwiki = True)) + raise NoPage(u'Image file URL for %s not found.' % self.aslink(forceInterwiki = True) ) return url def fileIsOnCommons(self): """Return True if the image is stored on Wikimedia Commons""" - return self.fileUrl().startswith( - u'http://upload.wikimedia.org/wikipedia/commons/') + return self.fileUrl().startswith(u'http://upload.wikimedia.org/wikipedia/commons/') def fileIsShared(self): """Return True if image is stored on Wikitravel shared repository.""" if 'wikitravel_shared' in self.site().shared_image_repository(): - return self.fileUrl().startswith( - u'http://wikitravel.org/upload/shared/') + return self.fileUrl().startswith(u'http://wikitravel.org/upload/shared/') return self.fileIsOnCommons() # FIXME: MD5 might be performed on not complete file due to server disconnection @@ -5325,25 +5322,26 @@ #else: if config.proxy['host']: conn = httplib.HTTPConnection(config.proxy['host']) + proxyPutAddr = '%s://%s%s' % (self.protocol(), self.hostname(), address) + conn.putrequest('POST', proxyPutAddr) + if type(config.proxy['auth']) == tuple: + import base64 + authcode = base64.b64encode("%s:%s" % (config.proxy['auth'][0], config.proxy['auth'][1]) ) + conn.putheader('Proxy-Authorization', "Basic %s" % authcode ) + else: if self.protocol() == 'http': conn = httplib.HTTPConnection(self.hostname()) elif self.protocol() == 'https': conn = httplib.HTTPSConnection(self.hostname()) + + conn.putrequest('POST', address) + # Encode all of this into a HTTP request # otherwise, it will crash, as other protocols are not supported if address[-1] == "?": address = address[:-1] - if config.proxy['host']: - proxyPutAddr = '%s://%s%s' % (self.protocol(), self.hostname(), address) - conn.putrequest('POST', proxyPutAddr) - if type(config.proxy['auth']) == tuple: - import base64 - authcode = base64.b64encode("%s:%s" % (config.proxy['auth'][0], config.proxy['auth'][1]) ) - conn.putheader('Proxy-Authorization', "Basic %s" % authcode ) - else: - conn.putrequest('POST', address) if self.hostname() in config.authenticate.keys(): import base64 authcode = base64.b64encode("%s:%s" % (config.authenticate[self.hostname()][0], config.authenticate[self.hostname()][1]) ) @@ -5405,32 +5403,8 @@ """ if retry is None: - retry=config.retry_on_fail + retry = config.retry_on_fail - #if False: #self.persistent_http and not data: - # self.conn.putrequest('GET', path) - # self.conn.putheader('User-agent', useragent) - # self.conn.putheader('Cookie', self.cookies(sysop = sysop)) - # self.conn.putheader('Connection', 'Keep-Alive') - # if compress: - # self.conn.putheader('Accept-encoding', 'gzip') - # self.conn.endheaders() - - # # Prepare the return values - # # Note that this can raise network exceptions which are not - # # caught here. - # try: - # response = self.conn.getresponse() - # except httplib.BadStatusLine: - # # Blub. - # self.conn.close() - # self.conn.connect() - # return self.getUrl(path, retry, sysop, data, compress, back_response=back_response) - - # text = response.read() - # headers = dict(response.getheaders()) - - #else: if self.hostname() in config.authenticate.keys(): uo = authenticateURLopener else: @@ -5459,10 +5433,10 @@ while True: try: if self.hostname() in config.authenticate.keys(): - request = urllib2.Request(url, data) - request.add_header('User-agent', useragent) - opener = urllib2.build_opener() - f = opener.open(request) + request = urllib2.Request(url, data) + request.add_header('User-agent', useragent) + opener = urllib2.build_opener() + f = opener.open(request) else: f = uo.open(url, data) @@ -5533,7 +5507,7 @@ self._getUserDataOld(text, sysop = sysop) if back_response: - return response, text + return f, text else: return text

1 0

SVN: [7492] trunk/pywikipedia/imageharvest.py
by a_engels＠svn.wikimedia.org 17 Oct '09

17 Oct '09

Revision: 7492 Author: a_engels Date: 2009-10-17 18:34:18 +0000 (Sat, 17 Oct 2009) Log Message: ----------- Categories on separate lines Modified Paths: -------------- trunk/pywikipedia/imageharvest.py Modified: trunk/pywikipedia/imageharvest.py =================================================================== --- trunk/pywikipedia/imageharvest.py 2009-10-17 17:51:49 UTC (rev 7491) +++ trunk/pywikipedia/imageharvest.py 2009-10-17 18:34:18 UTC (rev 7492) @@ -97,7 +97,7 @@ categories.append("[["+cat+"]]") else: categories.append("[["+mysite.namespace(14)+":"+cat+"]]") - desc = desc + "\r\n\r\n" + basicdesc + "\r\n\r\n" + " ".join(categories) + desc = desc + "\r\n\r\n" + basicdesc + "\r\n\r\n" + "\r\n".join(categories) uploadBot = upload.UploadRobot(image, description = desc) uploadBot.run() elif answer == 's':

1 0

SVN: [7491] trunk/pywikipedia/checkimages.py
by alexsh＠svn.wikimedia.org 17 Oct '09

17 Oct '09

Revision: 7491 Author: alexsh Date: 2009-10-17 17:51:49 +0000 (Sat, 17 Oct 2009) Log Message: ----------- main().takesettings(): make regex to view easy Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2009-10-17 16:26:48 UTC (rev 7490) +++ trunk/pywikipedia/checkimages.py 2009-10-17 17:51:49 UTC (rev 7491) @@ -649,7 +649,7 @@ # You can use this function also to find only the user that # has upload the image (FixME: Rewrite a bit this part) if put: - reportPageObject.put(reportPageText + self.newtext, comment = self.commImage, minorEdit = True) + reportPageObject.put(reportPageText + self.newtext, comment = self.commImage) # paginetta it's the image page object. try: if reportPageObject == self.image and self.uploader: @@ -1058,8 +1058,14 @@ self.settingsData = list() try: testo = wikiPage.get() - rxp = r"<------- ------->\n\*[Nn]ame ?= ?['\"](.*?)['\"]\n\*([Ff]ind|[Ff]indonly)=(.*?)\n\*[Ii]magechanges=(.*?)\n\*[Ss]ummary=['\"](.*?)['\"]\n\*[Hh]ead=['\"](.*?)['\"]\n\*[Tt]ext ?= ?['\"](.*?)['\"]\n\*[Mm]ex ?= ?['\"]?([^\n]*?)['\"]?\n" - r = re.compile(rxp, re.UNICODE|re.DOTALL) + r = re.compile(r"<------- ------->\n" + "\*[Nn]ame ?= ?['\"](.*?)['\"]\n" + "\*([Ff]ind|[Ff]indonly)=(.*?)\n" + "\*[Ii]magechanges=(.*?)\n" + "\*[Ss]ummary=['\"](.*?)['\"]\n" + "\*[Hh]ead=['\"](.*?)['\"]\n" + "\*[Tt]ext ?= ?['\"](.*?)['\"]\n" + "\*[Mm]ex ?= ?['\"]?([^\n]*?)['\"]?\n", re.UNICODE|re.DOTALL) number = 1 for m in r.finditer(testo):

1 0

SVN: [7490] trunk/pywikipedia/checkimages.py
by alexsh＠svn.wikimedia.org 17 Oct '09

17 Oct '09

Revision: 7490 Author: alexsh Date: 2009-10-17 16:26:48 +0000 (Sat, 17 Oct 2009) Log Message: ----------- remove namespace when use wp.ImagePage() load. (not necessary) Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2009-10-17 14:57:51 UTC (rev 7489) +++ trunk/pywikipedia/checkimages.py 2009-10-17 16:26:48 UTC (rev 7490) @@ -555,7 +555,7 @@ """ Function to set parameters, now only image but maybe it can be used for others in "future" """ self.imageName = imageName # Defing the image's Page Object - self.image = wikipedia.ImagePage(self.site, u'%s%s' % (self.image_namespace, self.imageName)) + self.image = wikipedia.ImagePage(self.site, self.imageName) self.timestamp = timestamp self.uploader = uploader @@ -638,7 +638,7 @@ """ Function to add the template in the image and to find out who's the user that has uploaded the file. """ # Get the image's description - reportPageObject = wikipedia.ImagePage(self.site, self.image_namespace + self.image_to_report) + reportPageObject = wikipedia.ImagePage(self.site, self.image_to_report) try: reportPageText = reportPageObject.get() @@ -801,7 +801,7 @@ max_usage = 0 for element in listGiven: imageName = element[1] - imagePage = wikipedia.ImagePage(self.site, u'File:%s' % imageName) + imagePage = wikipedia.ImagePage(self.site, imageName) imageUsage = [page for page in imagePage.usingPages()] if len(imageUsage) > 0 and len(imageUsage) > max_usage: max_usage = len(imageUsage) @@ -895,7 +895,7 @@ dupComment_talk = wikipedia.translate(self.site, duplicates_comment_talk) dupComment_image = wikipedia.translate(self.site, duplicates_comment_image) duplicateRegex = r'\[\[:File:%s\]\] has the following duplicates' % re.escape(self.convert_to_url(self.imageName)) - imagePage = wikipedia.ImagePage(self.site, u'File:%s' % self.imageName) + imagePage = wikipedia.ImagePage(self.site, self.imageName) hash_found = imagePage.getHash() duplicates = self.site.getFilesFromAnHash(hash_found) @@ -913,7 +913,7 @@ time_list = list() for duplicate in duplicates: - DupePage = wikipedia.ImagePage(self.site, u'File:%s' % duplicate) + DupePage = wikipedia.ImagePage(self.site, duplicate) if DupePage.urlname() == self.image.urlname() and self.timestamp != None: imagedata = self.timestamp @@ -926,15 +926,14 @@ time_list.append(data_seconds) older_image = self.returnOlderTime(time_image_list, time_list) # And if the images are more than two? - Page_oder_image = wikipedia.ImagePage(self.site, u'File:%s' % older_image) + Page_oder_image = wikipedia.ImagePage(self.site, older_image) string = '' images_to_tag_list = [] for duplicate in duplicates: - if wikipedia.ImagePage(self.site, u'%s:%s' % (self.image_namespace, duplicate)) == \ - wikipedia.ImagePage(self.site, u'%s:%s' % (self.image_namespace, older_image)): + if wikipedia.ImagePage(self.site, duplicate) == wikipedia.ImagePage(self.site, older_image): continue # the older image, not report also this as duplicate - DupePage = wikipedia.ImagePage(self.site, u'File:%s' % duplicate) + DupePage = wikipedia.ImagePage(self.site, duplicate) try: DupPageText = DupePage.get() older_page_text = Page_oder_image.get()

1 0

SVN: [7489] trunk/pywikipedia/featured.py
by xqt＠svn.wikimedia.org 17 Oct '09

17 Oct '09

Revision: 7489 Author: xqt Date: 2009-10-17 14:57:51 +0000 (Sat, 17 Oct 2009) Log Message: ----------- Don't process talk pages on el-wiki (bug #2114782) Modified Paths: -------------- trunk/pywikipedia/featured.py Modified: trunk/pywikipedia/featured.py =================================================================== --- trunk/pywikipedia/featured.py 2009-10-17 12:50:32 UTC (rev 7488) +++ trunk/pywikipedia/featured.py 2009-10-17 14:57:51 UTC (rev 7489) @@ -323,7 +323,7 @@ for p in raw: if p.namespace()==0: # Article arts.append(p) - elif p.namespace()==1: # Article talk (like in English) + elif p.namespace()==1 and site.lang <> 'el': # Article talk (like in English) arts.append(wikipedia.Page(p.site(), p.titleWithoutNamespace())) wikipedia.output('\03{lightred}** wikipedia:%s has %i %s articles\03{default}' % (site.lang, len(arts), feature)) return arts

1 0

SVN: [7488] trunk/pywikipedia/checkimages.py
by alexsh＠svn.wikimedia.org 17 Oct '09

17 Oct '09

Revision: 7488 Author: alexsh Date: 2009-10-17 12:50:32 +0000 (Sat, 17 Oct 2009) Log Message: ----------- syntax cleanup and make easy to view Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2009-10-17 12:09:43 UTC (rev 7487) +++ trunk/pywikipedia/checkimages.py 2009-10-17 12:50:32 UTC (rev 7488) @@ -276,27 +276,16 @@ # The message that the bot will add the second time that find another license problem. second_message_without_license = { - 'commons':None, - 'de':None, - 'en': None, - 'ga': None, + '_default':None, 'hu':u'\nSzia! Úgy tűnik a [[:Kép:%s]] képpel is hasonló a probléma, mint az előbbivel. Kérlek olvasd el a [[WP:KÉPLIC|feltölthető képek]]ről szóló oldalunk, és segítségért fordulj a [[WP:KF-JO|Jogi kocsmafalhoz]]. Köszönöm --~~~~', 'it':u':{{subst:Progetto:Coordinamento/Immagini/Bot/Messaggi/Senza licenza2|%s|__botnick__}} --~~~~', - 'ja':None, - 'ta':None, - 'zh':None, } # You can add some settings to wikipedia. In this way, you can change them without touching the code. # That's useful if you are running the bot on Toolserver. page_with_settings = { + '_default':None, 'commons':u'User:Filbot/Settings', - 'de':None, - 'en':None, - 'ga':None, - 'hu':None, 'it':u'Progetto:Coordinamento/Immagini/Bot/Settings#Settings', - 'ja':None, - 'ta':None, 'zh':u"User:Alexbot/cisettings#Settings", } # The bot can report some images (like the images that have the same name of an image on commons) @@ -367,38 +356,27 @@ } # A page where there's a list of template to skip. PageWithHiddenTemplates = { + '_default':None, 'commons': u'User:Filbot/White_templates#White_templates', - 'en':None, - 'ga':None, 'it':u'Progetto:Coordinamento/Immagini/Bot/WhiteTemplates', - 'ja':None, 'ko': u'User:Kwjbot_IV/whitetemplates/list', - 'ta':None, - 'zh':None, } # A page where there's a list of template to consider as licenses. PageWithAllowedTemplates = { + '_default':None, 'commons': u'User:Filbot/Allowed templates', - 'en':None, - 'ga':None, 'it':u'Progetto:Coordinamento/Immagini/Bot/AllowedTemplates', 'ko':u'User:Kwjbot_IV/AllowedTemplates', - 'ta':None, } # Template added when the bot finds only an hidden template and nothing else. # Note: every __botnick__ will be repleaced with your bot's nickname (feel free not to use if you don't need it) HiddenTemplateNotification = { + '_default':None, 'commons': u"""\n{{subst:User:Filnik/whitetemplate|File:%s}}\n\n''This message was '''added automatically by [[User:__botnick__|__botnick__]]''', if you need some help about it, ask its master (~~~) or go to the [[Commons:Help desk]]''. --~~~~""", - 'de' : None, - 'en' : None, - 'ga' : None, 'it' : u"{{subst:Progetto:Coordinamento/Immagini/Bot/Messaggi/Template_insufficiente|%s|__botnick__}} --~~~~", - 'ja' :None, 'ko' : u"\n{{subst:User:Kwj2772/whitetemplates|%s}} --~~~~", - 'ta' :None, - 'zh' :None, } # In this part there are the parameters for the dupe images. @@ -406,63 +384,38 @@ # Put here the template that you want to put in the image to warn that it's a dupe # put __image__ if you want only one image, __images__ if you want the whole list duplicatesText = { + '_default':None, 'commons': u'\n{{Dupe|__image__}}', - 'en' : None, - 'ga' : None, 'it' : u'\n{{Progetto:Coordinamento/Immagini/Bot/Template duplicati|__images__}}', - 'ja' :None, - 'ta' :None, - 'zh' :None, } # Head of the message given to the author duplicate_user_talk_head = { - 'commons': None, - 'en' : None, - 'ga' : None, + '_default':None, 'it' : u'\n\n== File doppio ==\n', - 'ja' :None, - 'ta' :None, - 'zh' :None, } # Message to put in the talk duplicates_user_talk_text = { + '_default':None, 'commons': u'{{subst:User:Filnik/duplicates|File:%s|File:%s}}', # FIXME: it doesn't exist - 'en' : None, - 'ga' : None, 'it' : u"{{subst:Progetto:Coordinamento/Immagini/Bot/Messaggi/Duplicati|%s|%s|__botnick__}} --~~~~", - 'ja' :None, - 'ta' :None, - 'zh' :None, } # Comment used by the bot while it reports the problem in the uploader's talk duplicates_comment_talk = { + '_default':None, 'commons': u'Bot: Dupe file found', - 'en' : None, - 'ga' : None, 'it' : u"Bot: Notifico il file doppio trovato", - 'ja' :None, - 'ta' :None, - 'zh' :None, } # Comment used by the bot while it reports the problem in the image duplicates_comment_image = { + '_default':None, 'commons': u'Bot: Tagging dupe file', - 'en' : None, - 'ga' : None, 'it' : u'Bot: File doppio, da cancellare', - 'ja' :None, - 'ta' :None, - 'zh' :None, } # Regex to detect the template put in the image's decription to find the dupe duplicatesRegex = { + '_default':None, 'commons': r'\{\{(?:[Tt]emplate:|)[Dd]upe[|}]', - 'en' : None, - 'ga' : None, 'it' : r'\{\{(?:[Tt]emplate:|)[Pp]rogetto:[Cc]oordinamento/Immagini/Bot/Template duplicati[|}]', - 'ja' :None, - 'ta' :None, - 'zh' :None, } # Category with the licenses and / or with subcategories with the other licenses. category_with_licenses = { @@ -480,29 +433,21 @@ ## is still None. # Page where is stored the message to send as email to the users emailPageWithText = { + '_default':None, 'de':'Benutzer:ABF/D3', - 'en':None, - 'ga':None, - 'ja':None, - 'ta':None, - 'zh':None, } # Title of the email emailSubject = { + '_default':None, 'de':'Problemen mit Deinem Bild auf der Deutschen Wikipedia', - 'en':None, - 'ga':None, - 'ja':None, - 'ta':None, - 'zh':None, } # Seems that uploaderBots aren't interested to get messages regarding the # files that they upload.. strange, uh? # Format: [[user,regex], [user,regex]...] the regex is needed to match the user where to send the warning-msg uploadBots = { + '_default':None, 'commons':[['File Upload Bot (Magnus Manske)',r'\|[Ss]ource=Transferred from .*?; transferred to Commons by \[\[User:(.*?)\]\]']], - 'en':None, } # Add your project (in alphabetical order) if you want that the bot start @@ -525,7 +470,6 @@ """ Function to print the messages followed by the TimeZone encoded correctly. """ if message[-1] != ' ': message = '%s ' % unicode(message) - time_zone = time.strftime("%d %b %Y %H:%M:%S (UTC)", time.localtime()) if locale.getlocale()[1]: time_zone = unicode(time.strftime(u"%d %b %Y %H:%M:%S (UTC)", time.gmtime()), locale.getlocale()[1]) else: @@ -534,7 +478,21 @@ class Global(object): # default environment settings - pass + # Command line configurable parameters + repeat = True # Restart after having check all the images? + limit = 80 # How many images check? + time_sleep = 30 # How many time sleep after the check? + skip_number = 0 # How many images to skip before checking? + waitTime = 0 # How many time sleep before the check? + commonsActive = False # Check if on commons there's an image with the same name? + normal = False # Check the new images or use another generator? + urlUsed = False # Use the url-related function instead of the new-pages generator + regexGen = False # Use the regex generator + untagged = False # Use the untagged generator + duplicatesActive = False # Use the duplicate option + duplicatesReport = False # Use the duplicate-report option + sendemailActive = False # Use the send-email + logFullError = True # Raise an error when the log is full # Here there is the main class. @@ -543,38 +501,56 @@ duplicatesReport = False, logFullError = True): """ Constructor, define some global variable """ self.site = site + self.logFullError = logFullError + self.logFulNumber = logFulNumber + self.settings = wikipedia.translate(self.site, page_with_settings) + self.rep_page = wikipedia.translate(self.site, report_page) + self.rep_text = wikipedia.translate(self.site, report_text) + self.com = wikipedia.translate(self.site, comm10) + hiddentemplatesRaw = wikipedia.translate(self.site, HiddenTemplate) - self.hiddentemplates = list() - for template in hiddentemplatesRaw: - self.hiddentemplates.append(wikipedia.Page(self.site, template)) + + self.hiddentemplates = [wikipedia.Page(self.site, tmp) for tmp in hiddentemplatesRaw] + self.pageHidden = wikipedia.translate(self.site, PageWithHiddenTemplates) + self.pageAllowed = wikipedia.translate(self.site, PageWithAllowedTemplates) # Commento = Summary in italian self.commento = wikipedia.translate(self.site, comm) # Adding the bot's nickname at the notification text if needed. botolist = wikipedia.translate(self.site, bot_list) + project = wikipedia.getSite().family.name + self.project = project + bot = config.usernames[project] try: botnick = bot[self.site.lang] except KeyError: raise wikipedia.NoUsername(u"You have to specify an username for your bot in this project in the user-config.py file.") + self.botnick = botnick botolist.append(botnick) + self.botolist = botolist + self.sendemailActive = sendemailActive - self.skip_list = list() # Inizialize the skip list used below + # Inizialize the skip list used below + self.skip_list = list() + self.duplicatesReport = duplicatesReport + self.image_namespace = u"File:" # Load the licenses only once, so do it once self.list_licenses = self.load_licenses() + def setParameters(self, imageName, timestamp, uploader): """ Function to set parameters, now only image but maybe it can be used for others in "future" """ self.imageName = imageName @@ -582,6 +558,7 @@ self.image = wikipedia.ImagePage(self.site, u'%s%s' % (self.image_namespace, self.imageName)) self.timestamp = timestamp self.uploader = uploader + def report(self, newtext, image_to_report, notification = None, head = None, notification2 = None, unver = True, commTalk = None, commImage = None): """ Function to make the reports easier. """ @@ -591,18 +568,22 @@ self.head = head self.notification = notification self.notification2 = notification2 - if self.notification != None: + + if self.notification: self.notification = re.sub(r'__botnick__', self.botnick, notification) - if self.notification2 != None: + + if self.notification2: self.notification2 = re.sub(r'__botnick__', self.botnick, notification2) self.commTalk = commTalk - if commImage == None: + + if commImage: + self.commImage = commImage + else: self.commImage = self.commento - else: - self.commImage = commImage + # Ok, done, let's loop. while 1: - if unver == True: + if unver: try: resPutMex = self.tag_image() except wikipedia.NoPage: @@ -612,7 +593,7 @@ wikipedia.output(u"Edit conflict! Skip!") break else: - if resPutMex == False: + if not resPutMex: break else: try: @@ -624,9 +605,10 @@ wikipedia.output(u"Edit conflict! Skip!") break else: - if resPutMex == False: + if not resPutMex: break - if self.notification != None and self.head != None: + + if self.notification and self.head: try: self.put_mex_in_talk() except wikipedia.EditConflict: @@ -640,21 +622,24 @@ break else: break - + def uploadBotChangeFunction(self, reportPageText, upBotArray): """ Detect the user that has uploaded the file through the upload bot """ regex = upBotArray[1] results = re.findall(regex, reportPageText) - if results == []: - return upBotArray[0] # we can't find the user, report the problem to the bot - else: + + if results: luser = results[0] return luser + else: + return upBotArray[0] # we can't find the user, report the problem to the bot + def tag_image(self, put = True): """ Function to add the template in the image and to find out who's the user that has uploaded the file. """ # Get the image's description reportPageObject = wikipedia.ImagePage(self.site, self.image_namespace + self.image_to_report) + try: reportPageText = reportPageObject.get() except wikipedia.NoPage: @@ -667,7 +652,7 @@ reportPageObject.put(reportPageText + self.newtext, comment = self.commImage, minorEdit = True) # paginetta it's the image page object. try: - if reportPageObject == self.image and self.uploader != None: + if reportPageObject == self.image and self.uploader: nick = self.uploader else: nick = reportPageObject.getLatestUploader()[0] @@ -679,7 +664,8 @@ return False upBots = wikipedia.translate(self.site, uploadBots) luser = wikipedia.url2link(nick, self.site, self.site) - if upBots != None: + + if upBots: for upBot in upBots: if upBot[0] == luser: luser = self.uploadBotChangeFunction(reportPageText, upBot) @@ -687,15 +673,16 @@ self.talk_page = talk_page self.luser = luser return True + def put_mex_in_talk(self): """ Function to put the warning in talk page of the uploader.""" commento2 = wikipedia.translate(self.site, comm2) emailPageName = wikipedia.translate(self.site, emailPageWithText) emailSubj = wikipedia.translate(self.site, emailSubject) - if self.notification2 == None: + if self.notification2: + self.notification2 = self.notification2 % self.image_to_report + else: self.notification2 = self.notification - else: - self.notification2 = self.notification2 % self.image_to_report second_text = False # Getting the talk page's history, to check if there is another advise... # The try block is used to prevent error if you use an old wikipedia.py's version. @@ -724,21 +711,23 @@ second_text = False ti_es_ti = wikipedia.translate(self.site, empty) testoattuale = ti_es_ti - if self.commTalk == None: + + if self.commTalk: + commentox = self.commTalk + else: commentox = commento2 + + if second_text: + newText = u"%s\n\n%s" % (testoattuale, self.notification2) else: - commentox = self.commTalk - if second_text == True: - try: - self.talk_page.put(u"%s\n\n%s" % (testoattuale, self.notification2), comment = commentox, minorEdit = False) - except wikipedia.LockedPage: - wikipedia.output(u'Talk page blocked, skip.') - elif second_text == False: - try: - self.talk_page.put(testoattuale + self.head + self.notification, comment = commentox, minorEdit = False) - except wikipedia.LockedPage: - wikipedia.output(u'Talk page blocked, skip.') - if emailPageName != None and emailSubj != None: + newText = testoattuale + self.head + self.notification + + try: + self.talk_page.put(newText, comment = commentox, minorEdit = False) + except wikipedia.LockedPage: + wikipedia.output(u'Talk page blocked, skip.') + + if emailPageName and emailSubj: emailPage = wikipedia.Page(self.site, emailPageName) try: emailText = emailPage.get() @@ -752,17 +741,19 @@ except userlib.UserActionRefuse: wikipedia.output("User is not mailable, aborted") return # exit - + def untaggedGenerator(self, untaggedProject, limit): """ Generator that yield the files without license. It's based on a tool of the toolserver. """ lang = untaggedProject.split('.', 1)[0] project = '.%s' % untaggedProject.split('.', 1)[1] + if lang == 'commons': link = 'http://toolserver.org/~daniel/WikiSense/UntaggedImages.php?wikifam=commons.…' else: link = 'http://toolserver.org/~daniel/WikiSense/UntaggedImages.php?wikilang=%s&wiki…' % (lang, project, limit, limit) text = self.site.getUrl(link, no_hostname = True) results = re.findall(r"""<td valign='top' title='Name'><a href='http://.*?\.org/w/index\.php\?title=(.*?)'>.*?</a></td>""", text) + if results: for result in results: wikiPage = wikipedia.ImagePage(self.site, result) @@ -770,31 +761,34 @@ else: wikipedia.output(link) raise NothingFound(u'Nothing found! Try to use the tool by yourself to be sure that it works!') - + def regexGenerator(self, regexp, textrun): """ Generator used when an user use a regex parsing a page to yield the results """ regex = re.compile(r'%s' % regexp, re.UNICODE|re.DOTALL) results = regex.findall(textrun) for image in results: yield wikipedia.ImagePage(self.site, image) - + def loadHiddenTemplates(self): """ Function to load the white templates """ # A template as {{en is not a license! Adding also them in the whitelist template... for langK in wikipedia.Family(u'wikipedia').langs.keys(): self.hiddentemplates.append(wikipedia.Page(self.site, u'Template:%s' % langK)) + # The template #if: and #switch: aren't something to care about #self.hiddentemplates.extend([u'#if:', u'#switch:']) FIXME + # Hidden template loading if self.pageHidden: try: pageHiddenText = wikipedia.Page(self.site, self.pageHidden).get() except (wikipedia.NoPage, wikipedia.IsRedirectPage): pageHiddenText = '' + for element in self.load(pageHiddenText): self.hiddentemplates.append(wikipedia.Page(self.site, element)) return self.hiddentemplates - + def returnOlderTime(self, listGiven, timeListGiven): """ Get some time and return the oldest of them """ # print listGiven; print timeListGiven @@ -813,26 +807,30 @@ max_usage = len(imageUsage) num_older = num num += 1 + if num_older: return listGiven[num_older][1] + for element in listGiven: time = element[0] imageName = element[1] not_the_oldest = False + for time_selected in timeListGiven: if time > time_selected: not_the_oldest = True break + if not not_the_oldest: return imageName - + def convert_to_url(self, page): # Function stolen from wikipedia.py """The name of the page this Page refers to, in a form suitable for the URL of the page.""" title = page.replace(u" ", u"_") encodedTitle = title.encode(self.site.encoding()) return urllib.quote(encodedTitle) - + def countEdits(self, pagename, userlist): """ Function to count the edit of a user or a list of users in a page. """ # self.botolist @@ -841,20 +839,22 @@ page = wikipedia.Page(self.site, pagename) history = page.getVersionHistory() user_list = list() + for data in history: user_list.append(data[2]) number_edits = 0 + for username in userlist: number_edits += user_list.count(username) return number_edits - + def checkImageOnCommons(self): """ Checking if the file is on commons """ wikipedia.output(u'Checking if %s is on commons...' % self.imageName) commons_site = wikipedia.getSite('commons', 'commons') regexOnCommons = r"\[\[:File:%s\]\] is also on '''Commons''': \[\[commons:File:.*?\]\](?: $same name$|)$" % re.escape(self.imageName) hash_found = self.image.getHash() - if hash_found == None: + if not hash_found: return False # Problems? Yes! Image deleted, no hash found. Skip the image. else: commons_image_with_this_hash = commons_site.getFilesFromAnHash(hash_found) @@ -881,7 +881,7 @@ else: # Problems? No, return True return True - + def checkImageDuplicated(self, duplicates_rollback): """ Function to check the duplicated files. """ # {{Dupe|File:Blanche_Montel.jpg}} @@ -898,18 +898,23 @@ imagePage = wikipedia.ImagePage(self.site, u'File:%s' % self.imageName) hash_found = imagePage.getHash() duplicates = self.site.getFilesFromAnHash(hash_found) - if duplicates == None: + + if not duplicates: return False # Error, image deleted, no hash found. Skip the image. + if len(duplicates) > 1: if len(duplicates) == 2: wikipedia.output(u'%s has a duplicate! Reporting it...' % self.imageName) else: wikipedia.output(u'%s has %s duplicates! Reporting them...' % (self.imageName, len(duplicates) - 1)) - if not dupText == None and not dupRegex == None: + + if dupText and dupRegex: time_image_list = list() time_list = list() + for duplicate in duplicates: DupePage = wikipedia.ImagePage(self.site, u'File:%s' % duplicate) + if DupePage.urlname() == self.image.urlname() and self.timestamp != None: imagedata = self.timestamp else: @@ -924,6 +929,7 @@ Page_oder_image = wikipedia.ImagePage(self.site, u'File:%s' % older_image) string = '' images_to_tag_list = [] + for duplicate in duplicates: if wikipedia.ImagePage(self.site, u'%s:%s' % (self.image_namespace, duplicate)) == \ wikipedia.ImagePage(self.site, u'%s:%s' % (self.image_namespace, older_image)): @@ -934,7 +940,8 @@ older_page_text = Page_oder_image.get() except wikipedia.NoPage: continue # The page doesn't exists - if re.findall(dupRegex, DupPageText) == [] and re.findall(dupRegex, older_page_text) == []: + + if not re.findall(dupRegex, DupPageText) and not re.findall(dupRegex, older_page_text): wikipedia.output(u'%s is a duplicate and has to be tagged...' % duplicate) images_to_tag_list.append(duplicate) #if duplicate != duplicates[-1]: @@ -946,12 +953,13 @@ return True # Ok - No problem. Let's continue the checking phase older_image_ns = u'%s%s' % (self.image_namespace, older_image) # adding the namespace only_report = False # true if the image are not to be tagged as dupes - + # put only one image or the whole list according to the request if u'__images__' in dupText: text_for_the_report = re.sub(r'__images__', r'\n%s*[[:%s]]\n' % (string, older_image_ns), dupText) else: text_for_the_report = re.sub(r'__image__', r'%s' % older_image_ns, dupText) + # Two iteration: report the "problem" to the user only once (the last) if len(images_to_tag_list) > 1: for image_to_tag in images_to_tag_list[:-1]: @@ -964,6 +972,7 @@ text_for_the_report = re.sub(r'\n\*\[\[:%s\]\]' % re.escape(self.image_namespace + image_to_tag), '', text_for_the_report) self.report(text_for_the_report, image_to_tag, commImage = dupComment_image, unver = True) + if len(images_to_tag_list) != 0 and not only_report: already_reported_in_past = self.countEdits(u'File:%s' % images_to_tag_list[-1], self.botolist) image_to_resub = images_to_tag_list[-1] @@ -977,11 +986,13 @@ self.report(text_for_the_report, images_to_tag_list[-1], dupTalkText % (older_image_ns, string), dupTalkHead, commTalk = dupComment_talk, commImage = dupComment_image, unver = True) + if self.duplicatesReport or only_report: if only_report: repme = u"\n*[[:File:%s]] has the following duplicates ('''forced mode'''):" % self.convert_to_url(self.imageName) else: repme = u"\n*[[:File:%s]] has the following duplicates:" % self.convert_to_url(self.imageName) + for duplicate in duplicates: if self.convert_to_url(duplicate) == self.convert_to_url(self.imageName): continue # the image itself, not report also this as duplicate @@ -989,16 +1000,19 @@ result = self.report_image(self.imageName, self.rep_page, self.com, repme, addings = False, regex = duplicateRegex) if not result: return True # If Errors, exit (but continue the check) + if older_image != self.imageName: return False # The image is a duplicate, it will be deleted. So skip the check-part, useless return True # Ok - No problem. Let's continue the checking phase - + def report_image(self, image_to_report, rep_page = None, com = None, rep_text = None, addings = True, regex = None): """ Report the files to the report page when needed. """ if not rep_page: rep_page = self.rep_page + if not com: com = self.com + if not rep_text: rep_text = self.rep_text @@ -1012,6 +1026,7 @@ text_get = '' except wikipedia.IsRedirectPage: text_get = another_page.getRedirectTarget().get() + if len(text_get) >= self.logFulNumber: if self.logFullError: raise LogIsFull(u"The log page (%s) is full! Please delete the old files reported." % another_page.title()) @@ -1021,6 +1036,7 @@ # The talk page includes "_" between the two names, in this way i replace them to " " n = re.compile(regex, re.UNICODE|re.DOTALL) y = n.findall(text_get) + if y: wikipedia.output(u"%s is already in the report page." % image_to_report) reported = False @@ -1032,7 +1048,7 @@ wikipedia.output(u"...Reported...") reported = True return reported - + def takesettings(self): """ Function to take the settings from the wiki. """ try: @@ -1046,6 +1062,7 @@ rxp = r"<------- ------->\n\*[Nn]ame ?= ?['\"](.*?)['\"]\n\*([Ff]ind|[Ff]indonly)=(.*?)\n\*[Ii]magechanges=(.*?)\n\*[Ss]ummary=['\"](.*?)['\"]\n\*[Hh]ead=['\"](.*?)['\"]\n\*[Tt]ext ?= ?['\"](.*?)['\"]\n\*[Mm]ex ?= ?['\"]?([^\n]*?)['\"]?\n" r = re.compile(rxp, re.UNICODE|re.DOTALL) number = 1 + for m in r.finditer(testo): name = str(m.group(1)) find_tipe = str(m.group(2)) @@ -1058,6 +1075,7 @@ tupla = [number, name, find_tipe, find, imagechanges, summary, head, text, mexcatched] self.settingsData += [tupla] number += 1 + if self.settingsData == list(): wikipedia.output(u"You've set wrongly your settings, please take a look to the relative page. (run without them)") self.settingsData = None @@ -1069,14 +1087,18 @@ wikipedia.output(u'Problems with loading the settigs, run without them.') self.settingsData = None self.some_problem = False - if self.settingsData == []: + + if not self.settingsData: self.settingsData = None + # Real-Time page loaded - if self.settingsData != None: wikipedia.output(u'\t >> Loaded the real-time page... <<') + if self.settingsData: + wikipedia.output(u'\t >> Loaded the real-time page... <<') # No settings found, No problem, continue. - else: wikipedia.output(u'\t >> No additional settings found! <<') + else: + wikipedia.output(u'\t >> No additional settings found! <<') return self.settingsData # Useless, but it doesn't harm.. - + def load_licenses(self): """ Load the list of the licenses """ """ @@ -1098,7 +1120,7 @@ wikipedia.output(u'\n\t...Loading the licenses allowed...\n') list_licenses = catlib.categoryAllPageObjectsAPI(catName) wikipedia.output('') # blank line - + # Add the licenses set in the default page as licenses # to check if self.pageAllowed: @@ -1112,7 +1134,7 @@ if pageLicense not in list_licenses: list_licenses.append(pageLicense) # the list has wiki-pages return list_licenses - + def miniTemplateCheck(self, template): """ Is the template given in the licenses allowed or in the licenses to skip? @@ -1133,7 +1155,7 @@ else: self.whiteTemplatesFound = True return False - + def templateInList(self): """ The problem is the calls to the Mediawiki system because they can be pretty slow. @@ -1155,7 +1177,7 @@ break except wikipedia.NoPage: continue - + def smartDetection(self): """ The bot instead of checking if there's a simple template in the image's description, checks also if that template is a license or @@ -1170,11 +1192,14 @@ self.hiddentemplates = self.loadHiddenTemplates() self.licenses_found = self.image.getTemplates() templatesInTheImageRaw = regex_find_licenses.findall(self.imageCheckText) - if self.licenses_found == [] and templatesInTheImageRaw != []: + + if not self.licenses_found and templatesInTheImageRaw: raise wikipedia.Error("APIs seems down. No templates found with them but actually there are templates used in the image's page!") self.allLicenses = list() - if self.list_licenses == []: + + if not self.list_licenses: raise wikipedia.Error(u'No licenses allowed provided, add that option to the code to make the script working correctly') + # Found the templates ONLY in the image's description for template_selected in templatesInTheImageRaw: for templateReal in self.licenses_found: @@ -1187,28 +1212,32 @@ # for example: there's only self, and not GFDL and the other licenses. #if self.allLicenses == self.licenses_found and not dummy_edit and self.licenses_found != []: # wikipedia.output(u"Seems that there's a problem regarding the Job queue, trying with a dummy edit to solve the problem.") - # try: + # try: # self.imageCheckText = self.image.get() # self.image.put(self.imageCheckText, 'Bot: Dummy edit,if you see this comment write [[User talk:%s|here]].' % self.botnick) # except (wikipedia.NoPage, wikipedia.IsRedirectPage): # return (None, list()) # dummy_edit = True #else: - break + break + if self.licenses_found != []: self.templateInList() + if self.license_found == None and self.allLicenses != list(): # If only iterlist = self.AllLicenses if I remove something # from iterlist it will be remove from self.AllLicenses too iterlist = list(self.allLicenses) + for template in iterlist: try: template.pageAPInfo() except wikipedia.IsRedirectPage: template = template.getRedirectTarget() except wikipedia.NoPage: - self.allLicenses.remove(template) - if self.allLicenses != list(): + self.allLicenses.remove(template) + + if self.allLicenses != list(): self.license_found = self.allLicenses[0].title() self.some_problem = False # If it has "some_problem" it must check # the additional settings. @@ -1245,7 +1274,7 @@ wikipedia.output(u"Skipping the file...") self.some_problem = False return (self.license_found, self.whiteTemplatesFound) - + def load(self, raw): """ Load a list of object from a string using regex. """ list_loaded = list() @@ -1349,7 +1378,7 @@ else: wikipedia.output(u"The wait option is available only with the standard generator.") return generator - + def isTagged(self): """ Understand if a file is already tagged or not. """ TextFind = wikipedia.translate(self.site, txt_find) @@ -1365,7 +1394,7 @@ elif i.lower() in self.imageCheckText: return True return False # Nothing Found? Ok: False - + def findAdditionalProblems(self): # In every tupla there's a setting configuration for tupla in self.settingsData: @@ -1410,7 +1439,7 @@ self.summary_used = summary self.mex_used = mexCatched continue - + def checkStep(self): # nothing = Defining an empty image description nothing = ['', ' ', ' ', ' ', '\n', '\n ', '\n ', '\n\n', '\n \n', ' \n', ' \n ', ' \n \n'] @@ -1509,6 +1538,8 @@ self.report(self.unvertext, self.imageName, notification, head, smwl) return True +gbv = Global() + def checkbot(): """ Main function """ # Command line configurable parameters @@ -1574,7 +1605,7 @@ firstPageTitle = wikipedia.input(u'From witch page do you want to start?') elif len(arg) > 6: firstPageTitle = arg[7:] - firstPageTitle = firstPageTitle.replace("File:", '').replace("file:", "") + firstPageTitle = firstPageTitle.split(":")[1:] generator = wikipedia.getSite().allpages(start=firstPageTitle, namespace=6) repeat = False elif arg.startswith('-page'): @@ -1741,10 +1772,11 @@ if repeat == True: printWithTimeZone(u"Waiting for %s seconds," % time_sleep) time.sleep(time_sleep) - elif repeat == False: + else: wikipedia.output(u"\t\t\t>> STOP! <<") break # Exit + # Here there is the main loop. I'll take all the (name of the) images and then i'll check them. if __name__ == "__main__": old = datetime.datetime.strptime(str(datetime.datetime.utcnow()).split('.')[0], "%Y-%m-%d %H:%M:%S") #timezones are UTC

1 0

SVN: [7487] trunk/pywikipedia/standardize_notes.py
by xqt＠svn.wikimedia.org 17 Oct '09

17 Oct '09

Revision: 7487 Author: xqt Date: 2009-10-17 12:09:43 +0000 (Sat, 17 Oct 2009) Log Message: ----------- solve some identation errors coming with r6540 (bug #2840435) Modified Paths: -------------- trunk/pywikipedia/standardize_notes.py Modified: trunk/pywikipedia/standardize_notes.py =================================================================== --- trunk/pywikipedia/standardize_notes.py 2009-10-17 11:10:07 UTC (rev 7486) +++ trunk/pywikipedia/standardize_notes.py 2009-10-17 12:09:43 UTC (rev 7487) @@ -46,7 +46,7 @@ # 2005-07-15: Build list of all sections which may contain citations: doFindAllCitationSections(). (SEWilco) # -from __future__ import generators +#from __future__ import generators import subprocess, sys, re, random import socket, urllib, robotparser import wikipedia, pagegenerators, config @@ -378,7 +378,7 @@ wikipedia.output( u"Reading existing Notes section" ) self.doReadReferencesSection( new_text, refsectionname ) while self.references and self.references[len(self.references)-1] == u'\n': - del self.references[len(self.references)-1] # delete trailing empty lines + del self.references[len(self.references)-1] # delete trailing empty lines # Convert any external links to footnote references wikipedia.output( u"Converting external links" ) new_text = self.doConvertExternalLinks( new_text ) @@ -416,53 +416,53 @@ new_text = new_text + text_line # skip section, so retain text. else: # TODO: recognize {{inline}} invisible footnotes when something can be done with them - # - # Ignore lines within comments - if not text_line.startswith( u'<!--' ): - # Fix erroneous external links in double brackets - Rextlink = re.compile(r'(?i)\[\[(?P<linkname>http://[^\]]+?)\]\]') - # TODO: compiling the regex each time might be inefficient - text_lineR = re.compile(Rextlink) - MOextlink = text_lineR.search(text_line) - while MOextlink: # find all links on line - extlink_linkname = MOextlink.group('linkname') - # Rewrite double brackets to single ones - text_line=text_line[:MOextlink.start()] + '[%s]' % extlink_linkname + text_line[MOextlink.end(0):] - MOextlink = text_lineR.search(text_line,MOextlink.start(0)+1) - # Regular expression to look for external link [linkname linktext] - linktext is optional. - # Also accepts erroneous pipe symbol as separator. - # Accepts wikilinks within <linktext> - #Rextlink = re.compile(r'[^\[]\[(?P<linkname>[h]*[ft]+tp:[^ [\]\|]+?)(?P<linktext>[ \|]+(( *[^\]\|]*)|( *\[\[.+?\]\])*)+)*\][^\]]') - #Rextlink = re.compile(r'\[(?P<linkname>[h]*[ft]+tp:[^ [\]\|]+?)(?P<linktext>[ \|]+(( *[^\]\|]*)|( *\[\[.+?\]\])*)+)*\]') - Rextlink = re.compile(r'(?i)\[(?P<linkname>[h]*[ft]+tp:[^ [\]\|]+?)(?P<linktext>[ \|]+(( *[^\]\|]*)|( *\[\[.+?\]\])*)+)*\]') - # TODO: compiling the regex each time might be inefficient - text_lineR = re.compile(Rextlink) - MOextlink = text_lineR.search(text_line) - while MOextlink: # find all links on line - extlink_linkname = MOextlink.group('linkname') - extlink_linktext = MOextlink.group('linktext') - self.refsequence += 1 - ( refname, reftext ) = self.doConvertLinkTextToReference(self.refsequence, extlink_linkname, extlink_linktext) - self.references.append( reftext ) # append new entry to References - if extlink_linktext: - # If there was text as part of link, reinsert text before footnote. - text_line=text_line[:MOextlink.start(0)] + '%s{{ref|%s}}' % (extlink_linktext, refname) + text_line[MOextlink.end(0):] - else: - text_line=text_line[:MOextlink.start(0)] + '{{ref|%s}}' % refname + text_line[MOextlink.end(0):] - MOextlink = text_lineR.search(text_line,MOextlink.start(0)+1) - # Search for {{doi}} - Rdoi = re.compile(r'(?i){{doi\|(?P<doilink>[^}|]*)}}') - # TODO: compiling the regex each time might be inefficient - doiR = re.compile(Rdoi) - MOdoi = doiR.search(text_line) - while MOdoi: # find all doi on line - doi_link = MOdoi.group('doilink') - if doi_link: - self.refsequence += 1 - ( refname, reftext ) = self.doConvertDOIToReference( self.refsequence, doi_link ) - self.references.append( reftext ) # append new entry to References - text_line=text_line[:MOdoi.start(0)] + '{{ref|%s}}' % refname + text_line[MOdoi.end(0):] - MOdoi = doiR.search(text_line, MOdoi.start(0)+1) + # + # Ignore lines within comments + if not text_line.startswith( u'<!--' ): + # Fix erroneous external links in double brackets + Rextlink = re.compile(r'(?i)\[\[(?P<linkname>http://[^\]]+?)\]\]') + # TODO: compiling the regex each time might be inefficient + text_lineR = re.compile(Rextlink) + MOextlink = text_lineR.search(text_line) + while MOextlink: # find all links on line + extlink_linkname = MOextlink.group('linkname') + # Rewrite double brackets to single ones + text_line=text_line[:MOextlink.start()] + '[%s]' % extlink_linkname + text_line[MOextlink.end(0):] + MOextlink = text_lineR.search(text_line,MOextlink.start(0)+1) + # Regular expression to look for external link [linkname linktext] - linktext is optional. + # Also accepts erroneous pipe symbol as separator. + # Accepts wikilinks within <linktext> + #Rextlink = re.compile(r'[^\[]\[(?P<linkname>[h]*[ft]+tp:[^ [\]\|]+?)(?P<linktext>[ \|]+(( *[^\]\|]*)|( *\[\[.+?\]\])*)+)*\][^\]]') + #Rextlink = re.compile(r'\[(?P<linkname>[h]*[ft]+tp:[^ [\]\|]+?)(?P<linktext>[ \|]+(( *[^\]\|]*)|( *\[\[.+?\]\])*)+)*\]') + Rextlink = re.compile(r'(?i)\[(?P<linkname>[h]*[ft]+tp:[^ [\]\|]+?)(?P<linktext>[ \|]+(( *[^\]\|]*)|( *\[\[.+?\]\])*)+)*\]') + # TODO: compiling the regex each time might be inefficient + text_lineR = re.compile(Rextlink) + MOextlink = text_lineR.search(text_line) + while MOextlink: # find all links on line + extlink_linkname = MOextlink.group('linkname') + extlink_linktext = MOextlink.group('linktext') + self.refsequence += 1 + ( refname, reftext ) = self.doConvertLinkTextToReference(self.refsequence, extlink_linkname, extlink_linktext) + self.references.append( reftext ) # append new entry to References + if extlink_linktext: + # If there was text as part of link, reinsert text before footnote. + text_line=text_line[:MOextlink.start(0)] + '%s{{ref|%s}}' % (extlink_linktext, refname) + text_line[MOextlink.end(0):] + else: + text_line=text_line[:MOextlink.start(0)] + '{{ref|%s}}' % refname + text_line[MOextlink.end(0):] + MOextlink = text_lineR.search(text_line,MOextlink.start(0)+1) + # Search for {{doi}} + Rdoi = re.compile(r'(?i){{doi\|(?P<doilink>[^}|]*)}}') + # TODO: compiling the regex each time might be inefficient + doiR = re.compile(Rdoi) + MOdoi = doiR.search(text_line) + while MOdoi: # find all doi on line + doi_link = MOdoi.group('doilink') + if doi_link: + self.refsequence += 1 + ( refname, reftext ) = self.doConvertDOIToReference( self.refsequence, doi_link ) + self.references.append( reftext ) # append new entry to References + text_line=text_line[:MOdoi.start(0)] + '{{ref|%s}}' % refname + text_line[MOdoi.end(0):] + MOdoi = doiR.search(text_line, MOdoi.start(0)+1) new_text = new_text + text_line # append new line to new text if new_text == '': new_text = original_text # If somehow no new text, return original text

1 0

← Newer
1
...
6
7
8
9
10
11
12
...
24
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn October 2009