Pywikipedia-svn October 2009

pywikipedia-svn@lists.wikimedia.org

10 participants
237 discussions

SVN: [7486] trunk/pywikipedia/commonscat.py
by xqt＠svn.wikimedia.org 17 Oct '09

17 Oct '09

Revision: 7486 Author: xqt Date: 2009-10-17 11:10:07 +0000 (Sat, 17 Oct 2009) Log Message: ----------- ignoreTemplate for ru-wiki #2794289 Modified Paths: -------------- trunk/pywikipedia/commonscat.py Modified: trunk/pywikipedia/commonscat.py =================================================================== --- trunk/pywikipedia/commonscat.py 2009-10-17 06:15:25 UTC (rev 7485) +++ trunk/pywikipedia/commonscat.py 2009-10-17 11:10:07 UTC (rev 7486) @@ -174,6 +174,7 @@ 'nds-nl' : [u'Commons'], 'nl' : [u'Commons', u'Commonsklein', u'Commonscatklein', u'Catbeg', u'Catsjab', u'Catwiki'], 'om' : [u'Commons'], + 'ru' : [u'Навигация'], } def getTemplate (lang = None):

1 0

SVN: [7485] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 16 Oct '09

16 Oct '09

Revision: 7485 Author: alexsh Date: 2009-10-17 06:15:25 +0000 (Sat, 17 Oct 2009) Log Message: ----------- add command and code cleanup Modified Paths: -------------- trunk/pywikipedia/checkimages.py trunk/pywikipedia/welcome.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2009-10-17 06:04:09 UTC (rev 7484) +++ trunk/pywikipedia/checkimages.py 2009-10-17 06:15:25 UTC (rev 7485) @@ -532,6 +532,11 @@ time_zone = unicode(time.strftime(u"%d %b %Y %H:%M:%S (UTC)", time.gmtime())) wikipedia.output(u"%s%s" % (message, time_zone)) +class Global(object): + # default environment settings + pass + + # Here there is the main class. class main: def __init__(self, site, logFulNumber = 25000, sendemailActive = False, @@ -757,15 +762,14 @@ else: link = 'http://toolserver.org/~daniel/WikiSense/UntaggedImages.php?wikilang=%s&wiki…' % (lang, project, limit, limit) text = self.site.getUrl(link, no_hostname = True) - regexp = r"""<td valign='top' title='Name'><a href='http://.*?\.org/w/index\.php\?title=(.*?)'>.*?</a></td>""" - results = re.findall(regexp, text) - if results == []: - wikipedia.output(link) - raise NothingFound(u'Nothing found! Try to use the tool by yourself to be sure that it works!') - else: + results = re.findall(r"""<td valign='top' title='Name'><a href='http://.*?\.org/w/index\.php\?title=(.*?)'>.*?</a></td>""", text) + if results: for result in results: wikiPage = wikipedia.ImagePage(self.site, result) yield wikiPage + else: + wikipedia.output(link) + raise NothingFound(u'Nothing found! Try to use the tool by yourself to be sure that it works!') def regexGenerator(self, regexp, textrun): """ Generator used when an user use a regex parsing a page to yield the results """ @@ -782,7 +786,7 @@ # The template #if: and #switch: aren't something to care about #self.hiddentemplates.extend([u'#if:', u'#switch:']) FIXME # Hidden template loading - if self.pageHidden != None: + if self.pageHidden: try: pageHiddenText = wikipedia.Page(self.site, self.pageHidden).get() except (wikipedia.NoPage, wikipedia.IsRedirectPage): @@ -809,7 +813,7 @@ max_usage = len(imageUsage) num_older = num num += 1 - if num_older != None: + if num_older: return listGiven[num_older][1] for element in listGiven: time = element[0] @@ -819,7 +823,7 @@ if time > time_selected: not_the_oldest = True break - if not_the_oldest == False: + if not not_the_oldest: return imageName def convert_to_url(self, page): @@ -832,7 +836,7 @@ def countEdits(self, pagename, userlist): """ Function to count the edit of a user or a list of users in a page. """ # self.botolist - if type(userlist) == type(''): + if type(userlist) == str: userlist = [userlist] page = wikipedia.Page(self.site, pagename) history = page.getVersionHistory() @@ -990,12 +994,18 @@ return True # Ok - No problem. Let's continue the checking phase def report_image(self, image_to_report, rep_page = None, com = None, rep_text = None, addings = True, regex = None): - """ Function to report the files in the report page when needed. """ - if rep_page == None: rep_page = self.rep_page - if com == None: com = self.com - if rep_text == None: rep_text = self.rep_text + """ Report the files to the report page when needed. """ + if not rep_page: + rep_page = self.rep_page + if not com: + com = self.com + if not rep_text: + rep_text = self.rep_text + another_page = wikipedia.Page(self.site, rep_page) - if regex == None: regex = image_to_report + + if not regex: + regex = image_to_report try: text_get = another_page.get() except wikipedia.NoPage: @@ -1011,22 +1021,23 @@ # The talk page includes "_" between the two names, in this way i replace them to " " n = re.compile(regex, re.UNICODE|re.DOTALL) y = n.findall(text_get) - if y == []: + if y: + wikipedia.output(u"%s is already in the report page." % image_to_report) + reported = False + else: # Adding the log if addings: rep_text = rep_text % image_to_report # Adding the name of the image in the report if not done already another_page.put(text_get + rep_text, comment = com, minorEdit = False) wikipedia.output(u"...Reported...") reported = True - else: - wikipedia.output(u"%s is already in the report page." % image_to_report) - reported = False return reported def takesettings(self): """ Function to take the settings from the wiki. """ try: - if self.settings == None: self.settingsData = None + if not self.settings: + self.settingsData = None else: wikiPage = wikipedia.Page(self.site, self.settings) self.settingsData = list() @@ -1082,7 +1093,7 @@ list_licenses.extend(pages) """ catName = wikipedia.translate(self.site, category_with_licenses) - if catName == None: + if not catName: raise wikipedia.Error(u'No licenses allowed provided, add that option to the code to make the script working correctly') wikipedia.output(u'\n\t...Loading the licenses allowed...\n') list_licenses = catlib.categoryAllPageObjectsAPI(catName) @@ -1090,11 +1101,12 @@ # Add the licenses set in the default page as licenses # to check - if self.pageAllowed != None: + if self.pageAllowed: try: pageAllowedText = wikipedia.Page(self.site, self.pageAllowed).get() except (wikipedia.NoPage, wikipedia.IsRedirectPage): pageAllowedText = '' + for nameLicense in self.load(pageAllowedText): pageLicense = wikipedia.Page(self.site, nameLicense) if pageLicense not in list_licenses: @@ -1107,10 +1119,11 @@ This function check this. """ if template in self.list_licenses: # the list_licenses are loaded in the __init__ (not to load them multimple times) - self.license_selected = template.title().replace('Template:', '') + self.license_selected = template.titleWithoutNamespace() self.seems_ok = True self.license_found = self.license_selected # let the last "fake" license normally detected return True + if template in self.hiddentemplates: # if the whitetemplate is not in the images description, we don't care try: Modified: trunk/pywikipedia/welcome.py =================================================================== --- trunk/pywikipedia/welcome.py 2009-10-17 06:04:09 UTC (rev 7484) +++ trunk/pywikipedia/welcome.py 2009-10-17 06:15:25 UTC (rev 7485) @@ -410,6 +410,7 @@ # The page where the bot reads the real-time whitelist page. # (this parameter is optional). whitelist_pg = { + '_default': None, 'ar':u'Project:سجل الترحيب/قائمة بيضاء', 'en':u'User:Filnik/whitelist', 'ga':u'Project:Log fáilte/Bánliosta', @@ -481,6 +482,7 @@ if not globalvar.filtBadName: return False + #initialize blacklist if not hasattr(self, '_blacklist') or force: elenco = [ ' ano', ' anus', 'anal ', 'babies', 'baldracca', 'balle', 'bastardo', @@ -513,6 +515,8 @@ "vandal", " v.f. ", "v. fighter", "vandal f.", "vandal fighter", 'wales jimmy', "wheels", "wales", "www.", ] + + #blacklist from wikipage badword_page = wikipedia.Page(self.site, wikipedia.translate(self.site, bad_pag) ) list_loaded = list() if badword_page.exists(): @@ -525,10 +529,11 @@ del elenco, elenco_others, list_loaded if not hasattr(self, '_whitelist') or force: + #initialize whitelist whitelist_default = ['emiliano'] wtlpg = wikipedia.translate(self.site, whitelist_pg) list_white = list() - if wtlpg != None: + if wtlpg: whitelist_page = wikipedia.Page(self.site, wtlpg) if whitelist_page.exists(): wikipedia.output(u'\nLoading the whitelist from %s...' % self.site ) @@ -539,6 +544,7 @@ else: showStatus(4) wikipedia.output(u"WARNING: The whitelist hasn't been setted!") + # Join the whitelist words. self._whitelist = list_white + whitelist_default del list_white, whitelist_default

1 0

SVN: [7484] trunk/pywikipedia/wikipedia.py
by a_engels＠svn.wikimedia.org 16 Oct '09

16 Oct '09

Revision: 7484 Author: a_engels Date: 2009-10-17 06:04:09 +0000 (Sat, 17 Oct 2009) Log Message: ----------- When not finding a mediawiki message, first try reloading the messages Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-17 03:33:28 UTC (rev 7483) +++ trunk/pywikipedia/wikipedia.py 2009-10-17 06:04:09 UTC (rev 7484) @@ -5770,10 +5770,10 @@ # Token not found output(u'WARNING: Token not found on %s. You will not be able to edit any page.' % self) - def mediawiki_message(self, key): + def mediawiki_message(self, key, forceReload = False): """Return the MediaWiki message text for key "key" """ # Allmessages is retrieved once for all per created Site object - if not self._mediawiki_messages: + if (not self._mediawiki_messages) or forceReload: api = False if verbose: output( @@ -5878,8 +5878,10 @@ try: return self._mediawiki_messages[key] except KeyError: - raise KeyError("MediaWiki key '%s' does not exist on %s" - % (key, self)) + if not forceReload: + return self.mediawiki_message(key, True) + else: + raise KeyError("MediaWiki key '%s' does not exist on %s" % (key, self)) def has_mediawiki_message(self, key): """Return True iff this site defines a MediaWiki message for 'key'."""

1 0

SVN: [7483] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 16 Oct '09

16 Oct '09

Revision: 7483 Author: alexsh Date: 2009-10-17 03:33:28 +0000 (Sat, 17 Oct 2009) Log Message: ----------- code cleanup Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-17 03:17:12 UTC (rev 7482) +++ trunk/pywikipedia/wikipedia.py 2009-10-17 03:33:28 UTC (rev 7483) @@ -5984,7 +5984,6 @@ # TODO: Repeat mechanism doesn't make much sense as implemented; # should use both offset and limit parameters, and have an # option to fetch older rather than newer pages - # TODO: extract and return edit comment. seen = set() try: d = self.apipath() @@ -6000,23 +5999,17 @@ 'rctype': 'new', 'rcnamespace': namespace, 'rclimit': int(number), - 'rcprop': ['title','timestamp','sizes','user','comment'], + 'rcprop': ['ids','title','timestamp','sizes','user','comment'], 'rcshow': ['!bot','!redirect'], #'': '', } data = query.GetData(params, self)['query']['recentchanges'] for np in data: - date = np['timestamp'] - title = np['title'] - length = np['newlen'] - username = np['user'] - loggedIn = u'' - comment = np['comment'] - if title not in seen: - seen.add(title) - page = Page(self, title) - yield page, date, length, loggedIn, username, comment + if np['pageid'] not in seen: + seen.add(np['pageid']) + page = Page(self, np['title'], defaultNamespace=np['ns']) + yield page, np['timestamp'], np['timestamp'], u'', np['user'], np['comment'] else: path = self.newpages_address(n=number, namespace=namespace) # The throttling is important here, so always enabled. @@ -6231,31 +6224,26 @@ 'letype' :'upload', 'lelimit' :int(number), } - if lestart is not None: params['lestart'] = lestart - if leend is not None: params['leend'] = leend - if leuser is not None: params['leuser'] = leuser - if letitle is not None: params['letitle'] = letitle + if lestart: params['lestart'] = lestart + if leend: params['leend'] = leend + if leuser: params['leuser'] = leuser + if letitle: params['letitle'] = letitle while True: - try: - if self.versionnumber() >= 11: - imagesData = query.GetData(params, self, encodeTitle = False)['query']['logevents'] - else: - raise NotImplementedError("The site version is not support this action.") - except KeyError: #no 'query' or 'logevents', error occured - raise ServerError("The APIs don't return the data, the site may be down") - except NotImplementedError: - raise ServerError("The site version is not support this action.") + if self.versionnumber() >= 11: + imagesData = query.GetData(params, self, encodeTitle = False) + else: + raise NotImplementedError("The site version is not support this action.") + + if 'error' in imagesData: + raise RuntimeError('%s' % data['error']) - for imageData in imagesData: + for i in imagesData['query']['logevents']: comment = '' - if 'comment' in imageData: - comment = imageData['comment'] - pageid = imageData['pageid'] - title = imageData['title'] - timestamp = imageData['timestamp'] - ##logid = imageData['logid'] #no use current now - user = imageData['user'] - yield ImagePage(self, title), timestamp, user, comment + if 'comment' in i: + comment = i['comment'] + #pageid = i['pageid'] + ##logid = i['logid'] #no use current now + yield ImagePage(self, i['title']), i['timestamp'], i['user'], comment if not repeat: break @@ -6302,32 +6290,29 @@ 'rcprop' : ['user','comment','timestamp','title','ids','loginfo'], #','flags','sizes','redirect','patrolled'] 'rclimit' : int(number), } - if rcstart is not None: params['rcstart'] = rcstart - if rcend is not None: params['rcend'] = rcend - if rcshow is not None: params['rcshow'] = rcshow - if rctype is not None: params['rctype'] = rctype + if rcstart: params['rcstart'] = rcstart + if rcend: params['rcend'] = rcend + if rcshow: params['rcshow'] = rcshow + if rctype: params['rctype'] = rctype while True: data = query.GetData(params, self, encodeTitle = False) + if 'error' in data: + raise RuntimeError('%s' % data['error']) try: rcData = data['query']['recentchanges'] except KeyError: raise ServerError("The APIs don't return data, the site may be down") - for rcItem in rcData: - try: - comment = rcItem['comment'] - except KeyError: - comment = '' - try: - loginfo = rcItem['loginfo'] - except KeyError: - loginfo = '' + for i in rcData: + comment = '' + if 'comment' in i: + comment = i['comment'] + loginfo = '' + if 'loginfo' in i: + loginfo = i['loginfo'] # pageid = rcItem['pageid'] - title = rcItem['title'] - timestamp = rcItem['timestamp'] # logid = rcItem['logid'] - user = rcItem['user'] - yield Page(self, title), timestamp, user, comment, loginfo + yield Page(self, i['title'], defaultNamespace=i['ns']), i['timestamp'], i['user'], comment, loginfo if not repeat: break @@ -6722,11 +6707,9 @@ count += 1 if not siteurl in pages['title']: # the links themselves have similar form - if pages['title'] in cache: - continue - else: - cache.append(pages['title']) - yield Page(self, pages['title']) + if pages['pageid'] not in cache: + cache.append(pages['pageid']) + yield Page(self, pages['title'], defaultNamespace=pages['ns']) if count >= limit: break

1 0

SVN: [7482] trunk/pywikipedia/imageharvest.py
by a_engels＠svn.wikimedia.org 16 Oct '09

16 Oct '09

Revision: 7482 Author: a_engels Date: 2009-10-17 03:17:12 +0000 (Sat, 17 Oct 2009) Log Message: ----------- ask for categories separately Modified Paths: -------------- trunk/pywikipedia/imageharvest.py Modified: trunk/pywikipedia/imageharvest.py =================================================================== --- trunk/pywikipedia/imageharvest.py 2009-10-16 20:00:34 UTC (rev 7481) +++ trunk/pywikipedia/imageharvest.py 2009-10-17 03:17:12 UTC (rev 7482) @@ -89,8 +89,16 @@ answer = wikipedia.inputChoice(u'Include image %s?' % image, ['yes', 'no', 'stop'], ['y', 'N', 's'], 'N') if answer == 'y': desc = wikipedia.input(u"Give the description of this image:") - desc = desc + "\r\n\n\r" + basicdesc - uploadBot = upload.UploadRobot(image, desc) + categories = [] + while True: + cat = wikipedia.input(u"Specify a category (or press enter to end adding categories)") + if not cat.strip(): break + if ":" in cat: + categories.append("[["+cat+"]]") + else: + categories.append("[["+mysite.namespace(14)+":"+cat+"]]") + desc = desc + "\r\n\r\n" + basicdesc + "\r\n\r\n" + " ".join(categories) + uploadBot = upload.UploadRobot(image, description = desc) uploadBot.run() elif answer == 's': break

1 0

SVN: [7481] trunk/pywikipedia/userlib.py
by filnik＠svn.wikimedia.org 16 Oct '09

16 Oct '09

Revision: 7481 Author: filnik Date: 2009-10-16 20:00:34 +0000 (Fri, 16 Oct 2009) Log Message: ----------- Bugfix: a , has been forgotten Modified Paths: -------------- trunk/pywikipedia/userlib.py Modified: trunk/pywikipedia/userlib.py =================================================================== --- trunk/pywikipedia/userlib.py 2009-10-16 18:33:06 UTC (rev 7480) +++ trunk/pywikipedia/userlib.py 2009-10-16 20:00:34 UTC (rev 7481) @@ -179,7 +179,7 @@ return False - def sendMailOld(self, subject = u'', text = u'' ccMe = False): + def sendMailOld(self, subject = u'', text = u'', ccMe = False): addr = self.site().put_address('Special:EmailUser') predata = { "wpSubject" : subject,

1 0

SVN: [7480] trunk/pywikipedia/userlib.py
by alexsh＠svn.wikimedia.org 16 Oct '09

16 Oct '09

Revision: 7480 Author: alexsh Date: 2009-10-16 18:33:06 +0000 (Fri, 16 Oct 2009) Log Message: ----------- version syntax typo Modified Paths: -------------- trunk/pywikipedia/userlib.py Modified: trunk/pywikipedia/userlib.py =================================================================== --- trunk/pywikipedia/userlib.py 2009-10-16 18:26:57 UTC (rev 7479) +++ trunk/pywikipedia/userlib.py 2009-10-16 18:33:06 UTC (rev 7480) @@ -149,7 +149,7 @@ if not self.site().isAllowed('sendemail'): raise UserActionRefuse("You don't have permission to send mail") - if wikipedia.config.use_api and self.site().versionnumber() > 14: + if wikipedia.config.use_api and self.site().versionnumber() >= 14: pass # will handle NotImplementedError later else: return self.sendMailOld(subject, text, ccMe)

1 0

SVN: [7479] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 16 Oct '09

16 Oct '09

Revision: 7479 Author: alexsh Date: 2009-10-16 18:26:57 +0000 (Fri, 16 Oct 2009) Log Message: ----------- Move checkimages.EmailSender() functions detail to User().sendMailOld() Modified Paths: -------------- trunk/pywikipedia/checkimages.py trunk/pywikipedia/userlib.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2009-10-16 17:23:35 UTC (rev 7478) +++ trunk/pywikipedia/checkimages.py 2009-10-16 18:26:57 UTC (rev 7479) @@ -79,7 +79,7 @@ # import re, time, urllib, urllib2, os, locale, sys, datetime -import wikipedia, config, pagegenerators, catlib, query +import wikipedia, config, pagegenerators, catlib, query, userlib locale.setlocale(locale.LC_ALL, '') @@ -532,52 +532,6 @@ time_zone = unicode(time.strftime(u"%d %b %Y %H:%M:%S (UTC)", time.gmtime())) wikipedia.output(u"%s%s" % (message, time_zone)) -class EmailSender(wikipedia.Page): - """ Class to send emails through the Wikipedia's dedicated page. """ - def __init__(self, site, user): - self.wikisite = site - self.user = user - page_special_name = u'Special:EmailUser' - self.page_special_name = page_special_name - # Special:EmailUser/Filnik - page = '%s/%s' % (self.page_special_name, self.user) - self.page = page - wikipedia.Page.__init__(self, site, page, None, 0) - - def send(self, subject, text, prompt = True): - """ Send an email through wikipedia's page. """ - host = self.site().hostname() - address = '/w/index.php?title=%s&target=%s&action=submit' % (self.page_special_name, self.user) - # Getting the token. - token = self.site().getToken(self) - # Defing the predata. - predata = { - "wpSubject" : subject, - "wpText" : text, - 'wpSend' : "Send", - 'wpCCMe' : '0', - } - predata['wpEditToken'] = token - if self.site().hostname() in config.authenticate.keys(): - predata['Content-type'] = 'application/x-www-form-urlencoded' - predata['User-agent'] = wikipedia.useragent - data = self.site().urlEncode(predata) - response = urllib2.urlopen(urllib2.Request('http://' + self.site().hostname() + address, data)) - data = u'' - else: - response, data = self.site().postForm(address, predata, sysop = False) - if data: - if 'var wgAction = "success";' in data: - wikipedia.output(u'Email sent') - return True - else: - wikipedia.output(u'Email not sent') - return False - else: - wikipedia.output(u'No data found.') - return False - - # Here there is the main class. class main: def __init__(self, site, logFulNumber = 25000, sendemailActive = False, @@ -787,8 +741,12 @@ return # Exit if self.sendemailActive: text_to_send = re.sub(r'__user-nickname__', r'%s' % self.luser, emailText) - emailClass = EmailSender(self.site, self.luser) - emailClass.send(emailSubj, text_to_send) + emailClass = userlib.User(self.site, self.luser) + try: + emailClass.sendMail(emailSubj, text_to_send) + except userlib.UserActionRefuse: + wikipedia.output("User is not mailable, aborted") + return # exit def untaggedGenerator(self, untaggedProject, limit): """ Generator that yield the files without license. It's based on a tool of the toolserver. """ Modified: trunk/pywikipedia/userlib.py =================================================================== --- trunk/pywikipedia/userlib.py 2009-10-16 17:23:35 UTC (rev 7478) +++ trunk/pywikipedia/userlib.py 2009-10-16 18:26:57 UTC (rev 7479) @@ -149,6 +149,11 @@ if not self.site().isAllowed('sendemail'): raise UserActionRefuse("You don't have permission to send mail") + if wikipedia.config.use_api and self.site().versionnumber() > 14: + pass # will handle NotImplementedError later + else: + return self.sendMailOld(subject, text, ccMe) + params = { 'action': 'emailuser', 'target': self.name(), @@ -169,10 +174,45 @@ elif 'emailuser' in result: if result['emailuser']['result'] == 'Success': + wikipedia.output(u'Email sent.') return True return False + def sendMailOld(self, subject = u'', text = u'' ccMe = False): + addr = self.site().put_address('Special:EmailUser') + predata = { + "wpSubject" : subject, + "wpText" : text, + 'wpSend' : "Send", + 'wpCCMe' : '0', + } + if ccMe: + predata['wpCCMe'] = '1' + + predata['wpEditToken'] = self.site().getToken() + + if self.site().hostname() in wikipedia.config.authenticate.keys(): + predata['Content-type'] = 'application/x-www-form-urlencoded' + predata['User-agent'] = wikipedia.useragent + data = self.site().urlEncode(predata) + response = wikipedia.urllib2.urlopen(urllib2.Request('http://' + self.site().hostname() + addr, data)) + data = response.read() + else: + response, data = self.site().postForm(address, predata, sysop = False) + + if data: + if 'var wgAction = "success";' in data: + wikipedia.output(u'Email sent.') + return True + else: + wikipedia.output(u'Email not sent.') + return False + else: + wikipedia.output(u'No data found.') + return False + + def contributions(self, limit = 500, namespace = []): """ Yields pages that the user has edited, with an upper bound of ``limit''. Pages returned are not guaranteed to be unique

1 0

SVN: [7478] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 16 Oct '09

16 Oct '09

Revision: 7478 Author: alexsh Date: 2009-10-16 17:23:35 +0000 (Fri, 16 Oct 2009) Log Message: ----------- Page().getVersionHistory(): separate getdata part and add API Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-16 06:35:13 UTC (rev 7477) +++ trunk/pywikipedia/wikipedia.py 2009-10-16 17:23:35 UTC (rev 7478) @@ -2408,20 +2408,11 @@ # regular expression matching one edit in the version history. # results will have 4 groups: oldid, edit date/time, user name, and edit # summary. - if self.site().versionnumber() < 4: - editR = re.compile('<li>\(.*?\)\s+\(.*\).*?<a href=".*?oldid=([0-9]*)" title=".*?">([^<]*)</a> <a href=".*?" title=".*?">([^<]*?)</a>.*?(?:(.*?))?</li>') - elif self.site().versionnumber() < 15: - editR = re.compile('<li>\(.*?\)\s+\(.*\).*?<a href=".*?oldid=([0-9]*)" title=".*?">([^<]*)</a> (?:|)<a href=".*?" title=".*?">([^<]*?)</a>.*?(?:|).*?(?:(.*?))?</li>') - elif self.site().versionnumber() < 16: - editR = re.compile(r'<li class=".*?">\((?:\w*|<a[^<]*</a>)\)\s\((?:\w*|<a[^<]*</a>)\).*?<a href=".*?([0-9]*)" title=".*?">([^<]*)</a> <a [^>]*?>([^<]*?)</a>.*?(?: .*?|)(?: .*?|)(?: \((?:|)(.*?)(?:|)\))?(?: \(.*?\)|)\s*</li>', re.UNICODE) - else: - editR = re.compile(r'<li(?: class="mw-tag[^>]+)?>\((?:\w+|<a[^<]*</a>)\)\s\((?:\w+|<a[^<]*</a>)\).*?<a href=".*?([0-9]*)" title=".*?">([^<]*)</a> <a [^>]*?>([^<]*?)</a>.*?(?: <abbr class="minor"[^>]*?>.*?</abbr>|)(?: .*?|)(?: \((?:|)(.*?)(?:|)\))?(?: \(.*?\))?(?: .*?\))?\s*</li>', re.UNICODE) - startFromPage = None thisHistoryDone = False skip = False # Used in determining whether we need to skip the first page dataQuery = [] + hasData = False - RLinkToNextPage = re.compile('&offset=(.*?)&') # Are we getting by Earliest first? if reverseOrder: @@ -2444,9 +2435,98 @@ dataQuery = self._versionhistory else: thisHistoryDone = True + + if not thisHistoryDone: + dataQuery.extend(self._getVersionHistory(getAll, skip, reverseOrder, revCount)) + + if reverseOrder: + # Return only revCount edits, even if the version history is extensive + if dataQuery != []: + self._versionhistoryearliest = dataQuery + del dataQuery + if len(self._versionhistoryearliest) > revCount and not getAll: + return self._versionhistoryearliest[0:revCount] + return self._versionhistoryearliest + + if dataQuery != []: + self._versionhistory = dataQuery + del dataQuery + # Return only revCount edits, even if the version history is extensive + if len(self._versionhistory) > revCount and not getAll: + return self._versionhistory[0:revCount] + return self._versionhistory + + def _getVersionHistory(self, getAll = False, skipFirst = False, reverseOrder = False, + revCount=500): + """Load history informations by API query. + Internal use for self.getVersionHistory(), don't use this function directly. + """ + try: + if config.use_api and self.site().versionnumber() >= 8: + x = self.site().api_address() + del x + else: + raise NotImplementedError + except NotImplementedError: + return self._getVersionHistoryOld(reExist, getAll, skipFirst, reverseOrder, revCount) + dataQ = [] + thisHistoryDone = False + params = { + 'action': 'query', + 'prop': 'revisions', + 'titles': self.title(), + 'rvlimit': revCount, + #'': '', + } + while not thisHistoryDone: + if reverseOrder: + params['rvdir'] = 'newer' + + result = query.GetData(params, self.site()) + if 'error' in result: + raise RuntimeError("%s" % result['error']) + + if 'query-continue' in result and getAll: + params['rvstartid'] = result['query-continue']['revisions']['rvstartid'] + else: + thisHistoryDone = True + + if skipFirst: + skipFirst = False + else: + for r in result['query']['pages'].values()[0]['revisions']: + c = '' + if 'comment' in r: + c = r['comment'] + #revision id, edit date/time, user name, edit summary + dataQ.append((r['revid'], r['timestamp'], r['user'], c)) + + if len(result['query']['pages'].values()[0]['revisions']) < revCount: + thisHistoryDone = True + + return dataQ + + def _getVersionHistoryOld(self, getAll = False, skipFirst = False, + reverseOrder = False, revCount=500): + """Load the version history page and return history information. + Internal use for self.getVersionHistory(), don't use this function directly. + """ + dataQ = [] + thisHistoryDone = False + startFromPage = None + if self.site().versionnumber() < 4: + editR = re.compile('<li>\(.*?\)\s+\(.*\).*?<a href=".*?oldid=([0-9]*)" title=".*?">([^<]*)</a> <a href=".*?" title=".*?">([^<]*?)</a>.*?(?:(.*?))?</li>') + elif self.site().versionnumber() < 15: + editR = re.compile('<li>\(.*?\)\s+\(.*\).*?<a href=".*?oldid=([0-9]*)" title=".*?">([^<]*)</a> (?:|)<a href=".*?" title=".*?">([^<]*?)</a>.*?(?:|).*?(?:(.*?))?</li>') + elif self.site().versionnumber() < 16: + editR = re.compile(r'<li class=".*?">\((?:\w*|<a[^<]*</a>)\)\s\((?:\w*|<a[^<]*</a>)\).*?<a href=".*?([0-9]*)" title=".*?">([^<]*)</a> <a [^>]*?>([^<]*?)</a>.*?(?: .*?|)(?: .*?|)(?: \((?:|)(.*?)(?:|)\))?(?: \(.*?\)|)\s*</li>', re.UNICODE) + else: + editR = re.compile(r'<li(?: class="mw-tag[^>]+)?>\((?:\w+|<a[^<]*</a>)\)\s\((?:\w+|<a[^<]*</a>)\).*?<a href=".*?([0-9]*)" title=".*?">([^<]*)</a> <a [^>]*?>([^<]*?)</a>.*?(?: <abbr class="minor"[^>]*?>.*?</abbr>|)(?: .*?|)(?: \((?:|)(.*?)(?:|)\))?(?: \(.*?\))?(?: .*?\))?\s*</li>', re.UNICODE) + RLinkToNextPage = re.compile('&offset=(.*?)&') + while not thisHistoryDone: - path = self.site().family.version_history_address(self.site().language(), self.urlname(), revCount) + path = self.site().family.version_history_address(self.site().language(), self.urlname(), config.special_page_limit) if reverseOrder: path += '&dir=prev' @@ -2471,60 +2551,30 @@ # save a copy of the text self_txt = txt - # If we are getting all of the page history... - if getAll: - #Find the nextPage link, if not exist, the page is last history page - matchObj = RLinkToNextPage.search(self_txt) - if matchObj: - startFromPage = matchObj.group(1) - else: - thisHistoryDone = True + #Find the nextPage link, if not exist, the page is last history page + matchObj = RLinkToNextPage.search(self_txt) + if getAll and matchObj: + startFromPage = matchObj.group(1) + else: + thisHistoryDone = True - if len(dataQuery) == 0: - edits = editR.findall(self_txt) - if reverseOrder: - edits.reverse() - #for edit in edits: - dataQuery.extend([edit for edit in edits]) - if len(edits) < revCount: - thisHistoryDone = True - else: - if not skip: - edits = editR.findall(self_txt) - if reverseOrder: - edits.reverse() - #for edit in edits: - dataQuery.extend([edit for edit in edits]) - if len(edits) < revCount: - thisHistoryDone = True - else: - # Skip the first page only, - skip = False + if not skipFirst: + edits = editR.findall(self_txt) + + if skipFirst: + # Skip the first page only, + skipFirst = False else: - # If we are not getting all, we stop on the first page. - #for edit in editR.findall(self_txt): - dataQuery.extend([edit for edit in editR.findall(self_txt)] ) if reverseOrder: - dataQuery.reverse() - thisHistoryDone = True - - if reverseOrder: - # Return only revCount edits, even if the version history is extensive - if dataQuery != []: - self._versionhistoryearliest = dataQuery - del dataQuery - if len(self._versionhistoryearliest) > revCount and not getAll: - return self._versionhistoryearliest[0:revCount] - return self._versionhistoryearliest - - if dataQuery != []: - self._versionhistory = dataQuery - del dataQuery - # Return only revCount edits, even if the version history is extensive - if len(self._versionhistory) > revCount and not getAll: - return self._versionhistory[0:revCount] - return self._versionhistory - + edits.reverse() + #for edit in edits: + dataQ.extend(edits) + if len(edits) < revCount: + thisHistoryDone = True + + return dataQ + + def getVersionHistoryTable(self, forceReload=False, reverseOrder=False, getAll=False, revCount=500): """Return the version history as a wiki table."""

1 0

SVN: [7477] trunk/pywikipedia/checkimages.py
by xqt＠svn.wikimedia.org 15 Oct '09

15 Oct '09

Revision: 7477 Author: xqt Date: 2009-10-16 06:35:13 +0000 (Fri, 16 Oct 2009) Log Message: ----------- checkimages.py translation for Irish (ga) Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2009-10-16 01:35:28 UTC (rev 7476) +++ trunk/pywikipedia/checkimages.py 2009-10-16 06:35:13 UTC (rev 7477) @@ -46,12 +46,12 @@ -nologerror - If given, this option will disable the error that is risen when the log is full. ----- Istructions for the real-time settings ---- +---- Instructions for the real-time settings ---- * For every new block you have to add: <------- -------> -In this way the Bot can understand where the block start to take the right parameter. +In this way the Bot can understand where the block starts in order to take the right parameter. * Name= Set the name of the block * Find= Use it to define what search in the text of the image's description, @@ -97,6 +97,7 @@ 'de' :u'{{Benutzer:ABF/D|~~~~}} {{Dateiüberprüfung/benachrichtigt (Kategorie)|{{subst:LOCALYEAR}}|{{subst:LOCALMONTH}}|{{subst:LOCALDAY}}}} {{Dateiüberprüfung/benachrichtigt (Text)|Lizenz|||||}} --This was added by ~~~~-- ', 'en' :u'\n{{subst:nld}}', 'fr' :u'\n{{subst:lid}}', + 'ga' :u'\n{{subst:Ceadúnas de dhíth}}', 'hu' :u'\n{{nincslicenc|~~~~~}}', 'it' :u'\n{{subst:unverdata}}', 'ja' :u'{{subst:Nld}}', @@ -114,6 +115,7 @@ 'commons':[u'{{no license', u'{{no license/en', u'{{nld', u'{{no permission since'], 'de':[u'{{DÜP', u'{{Dateiüberprüfung'], 'en':[u'{{nld', u'{{no license'], + 'ga':[u'{{Ceadúnas de dhíth', u'{{Ceadúnas de dhíth'], 'hu':[u'{{nincsforrás',u'{{nincslicenc'], 'it':[u'{{unverdata', u'{{unverified'], 'ja':[u'{{no source', u'{{unknown', u'{{non free', u'\n{{subst:Benvebot}}\n~~~~\n', 'ja' :u'{{subst:Welcome/intro}}\n{{subst:welcome|--~~~~}}\n', @@ -157,6 +161,7 @@ 'commons':u"Bot: Requesting source information." , 'de' :u'Bot:Notify User', 'en' :u"Bot: Requesting source information." , + 'ga' :u"Róbó: Ag iarraidh eolais foinse." , 'it' :u"Bot: Notifico l'unverified", 'hu' :u'Robot: Forrásinformáció kérése', 'ja' :u"ロボットによる:著作権情報明記のお願い", @@ -171,6 +176,7 @@ 'commons':u"{{speedy|The file has .%s as extension. Is it ok? Please check.}}", 'ar' :u"{{شطب|الملف له .%s كامتداد.}}", 'en' :u"{{db-meta|The file has .%s as extension.}}", + 'ga' :u"{{scrios|Tá iarmhír .%s ar an comhad seo.}}", 'hu' :u'{{azonnali|A fájlnak .%s a kiterjesztése}}', 'it' :u'{{cancella subito|motivo=Il file ha come estensione ".%s"}}', 'ja' :u'{{db|知らないファイルフォーマット %s}}', @@ -184,6 +190,7 @@ 'commons':u"\n== Unknown extension! ==\n", 'ar' :u"\n== امتداد غير معروف! ==\n", 'en' :u"\n== Unknown extension! ==\n", + 'ga' :u"\n== Iarmhír neamhaithnid! ==\n", 'fr' :u'\n== Extension inconnue ==\n', 'hu' :u'\n== Ismeretlen kiterjesztésű fájl ==\n', 'it' :u'\n\n== File non specificato ==\n', @@ -197,6 +204,7 @@ 'ar' :u'الملف [[:File:%s]] يبدو أن امتداده خاطيء, من فضلك تحقق. ~~~~', 'commons':u'The [[:File:%s]] file seems to have a wrong extension, please check. ~~~~', 'en' :u'The [[:File:%s]] file seems to have a wrong extension, please check. ~~~~', + 'ga' :u'Tá iarmhír mícheart ar an comhad [[:File:%s]], scrúdaigh le d\'thoil. ~~~~', 'fr' :u'Le fichier [[:File:%s]] semble avoir une mauvaise extension, veuillez vérifier. ~~~~', 'hu' :u'A [[:Kép:%s]] fájlnak rossz a kiterjesztése, kérlek ellenőrízd. ~~~~', 'it' :u'{{subst:Progetto:Coordinamento/Immagini/Bot/Messaggi/Ext|%s|__botnick__}} --~~~~', @@ -204,11 +212,12 @@ 'ta' :u'[[:படிமம்:%s]] இனங்காணப்படாத கோப்பு நீட்சியை கொண்டுள்ளது தயவு செய்து ஒரு முறை சரி பார்க்கவும் ~~~~', 'zh' :u'您好，你上傳的[[:File:%s]]無法被識別，請檢查您的檔案，謝謝。--~~~~', } -# Summary of the delate immediately. (f.e: Adding {{db-meta|The file has .%s as extension.}}) +# Summary of the delete immediately. (f.e: Adding {{db-meta|The file has .%s as extension.}}) del_comm = { 'ar' :u'بوت: إضافة %s', 'commons':u'Bot: Adding %s', 'en' :u'Bot: Adding %s', + 'ga' :u'Róbó: Cuir %s leis', 'fr' :u'Robot : Ajouté %s', 'hu' :u'Robot:"%s" hozzáadása', 'it' :u'Bot: Aggiungo %s', @@ -225,6 +234,7 @@ 'commons':u"",# Nothing, the template has already the header inside. 'de' :u"\n== Bild ohne Lizenz ==\n", 'en' :u"\n== Image without license ==\n", + 'ga' :u"\n== Comhad gan ceadúnas ==\n", 'fr' :u"\n== Fichier sans licence ==\n", 'hu' :u"\n== Licenc nélküli kép ==\n", 'it' :u"\n\n== File senza licenza ==\n", @@ -241,6 +251,7 @@ 'ar' :u"{{subst:مصدر الصورة|File:%s}} --~~~~", 'de' :u'\n{{subst:Benutzer:ABF/D2|%s}} ~~~~ ', 'en' :u"{{subst:image source|File:%s}} --~~~~", + 'ga' :u"{{subst:Foinse na híomhá|File:%s}} --~~~~", 'hu' :u"{{subst:adjforrást|Kép:%s}} \n Ezt az üzenetet ~~~ automatikusan helyezte el a vitalapodon, kérdéseddel fordulj a gazdájához, vagy a [[WP:KF|Kocsmafalhoz]]. --~~~~", 'it' :u"{{subst:Progetto:Coordinamento/Immagini/Bot/Messaggi/Senza licenza|%s|__botnick__}} --~~~~", 'ja' :u"\n{{subst:Image copyright|File:%s}}--~~~~", @@ -255,6 +266,7 @@ 'commons':[u'Siebot', u'CommonsDelinker', u'Filbot', u'John Bot', u'Sz-iwbot', u'ABFbot'], 'de' :[u'ABFbot'], 'en' :[u'OrphanBot'], + 'ga' :[u'AllieBot'], 'it' :[u'Filbot', u'Nikbot', u'.snoopyBot.'], 'ja' :[u'Alexbot'], 'ko' :[u'Kwjbot IV'], @@ -267,18 +279,20 @@ 'commons':None, 'de':None, 'en': None, + 'ga': None, 'hu':u'\nSzia! Úgy tűnik a [[:Kép:%s]] képpel is hasonló a probléma, mint az előbbivel. Kérlek olvasd el a [[WP:KÉPLIC|feltölthető képek]]ről szóló oldalunk, és segítségért fordulj a [[WP:KF-JO|Jogi kocsmafalhoz]]. Köszönöm --~~~~', 'it':u':{{subst:Progetto:Coordinamento/Immagini/Bot/Messaggi/Senza licenza2|%s|__botnick__}} --~~~~', 'ja':None, 'ta':None, 'zh':None, } -# You can add some settings to wikipedia. In this way, you can change them without touch the code. +# You can add some settings to wikipedia. In this way, you can change them without touching the code. # That's useful if you are running the bot on Toolserver. page_with_settings = { 'commons':u'User:Filbot/Settings', 'de':None, 'en':None, + 'ga':None, 'hu':None, 'it':u'Progetto:Coordinamento/Immagini/Bot/Settings#Settings', 'ja':None, @@ -291,6 +305,7 @@ 'commons':u'User:Filbot/Report', 'de' :u'Benutzer:ABFbot/Report', 'en' :u'User:Filnik/Report', + 'ga' :u'User:AllieBot/ReportImages', 'hu' :u'User:Bdamokos/Report', 'it' :u'Progetto:Coordinamento/Immagini/Bot/Report', 'ja' :u'User:Alexbot/report', @@ -306,6 +321,7 @@ 'ar':u"\n*[[:صورة:%s]] " + timeselected, 'de':u"\n*[[:Bild:%s]] " + timeselected, 'en':u"\n*[[:File:%s]] " + timeselected, + 'ga':u"\n*[[:File:%s]] " + timeselected, 'hu':u"\n*[[:Kép:%s]] " + timeselected, 'it':u"\n*[[:File:%s]] " + timeselected, 'ja':u"\n*[[:File:%s]] " + timeselected, @@ -320,6 +336,7 @@ 'de' :u'Bot:schreibe Log', 'en' :u'Bot: Updating the log', 'fr' :u'Robot: Mise à jour du journal', + 'ga' :u'Róbó: Log a thabhairt suas chun dáta', 'hu' :u'Robot: A napló frissítése', 'it' :u'Bot: Aggiorno il log', 'ja' :u'ロボットによる:更新', @@ -329,7 +346,7 @@ } # If a template isn't a license but it's included on a lot of images, that can be skipped to -# analise the image without taking care of it. (the template must be in a list) +# analyze the image without taking care of it. (the template must be in a list) # Warning: Don't add template like "en, de, it" because they are already in (added in the code, below # Warning 2: The bot will use regex, make the names compatible, please (don't add "Template:" or {{ # because they are already put in the regex). @@ -340,6 +357,7 @@ 'de':[u'Template:Information'], 'en':[u'Template:Information'], 'fr':[u'Template:Information'], + 'ga':[u'Template:Information'], 'hu':[u'Template:Információ', u'Template:Enwiki', u'Template:Azonnali'], 'it':[u'Template:EDP', u'Template:Informazioni file', u'Template:Information', u'Template:Trademark', u'Template:Permissionotrs'], # Put the other in the page on the project defined below 'ja':[u'Template:Information'], @@ -351,6 +369,7 @@ PageWithHiddenTemplates = { 'commons': u'User:Filbot/White_templates#White_templates', 'en':None, + 'ga':None, 'it':u'Progetto:Coordinamento/Immagini/Bot/WhiteTemplates', 'ja':None, 'ko': u'User:Kwjbot_IV/whitetemplates/list', @@ -362,6 +381,7 @@ PageWithAllowedTemplates = { 'commons': u'User:Filbot/Allowed templates', 'en':None, + 'ga':None, 'it':u'Progetto:Coordinamento/Immagini/Bot/AllowedTemplates', 'ko':u'User:Kwjbot_IV/AllowedTemplates', 'ta':None, @@ -373,6 +393,7 @@ 'commons': u"""\n{{subst:User:Filnik/whitetemplate|File:%s}}\n\n''This message was '''added automatically by [[User:__botnick__|__botnick__]]''', if you need some help about it, ask its master (~~~) or go to the [[Commons:Help desk]]''. --~~~~""", 'de' : None, 'en' : None, + 'ga' : None, 'it' : u"{{subst:Progetto:Coordinamento/Immagini/Bot/Messaggi/Template_insufficiente|%s|__botnick__}} --~~~~", 'ja' :None, 'ko' : u"\n{{subst:User:Kwj2772/whitetemplates|%s}} --~~~~", @@ -387,6 +408,7 @@ duplicatesText = { 'commons': u'\n{{Dupe|__image__}}', 'en' : None, + 'ga' : None, 'it' : u'\n{{Progetto:Coordinamento/Immagini/Bot/Template duplicati|__images__}}', 'ja' :None, 'ta' :None, @@ -396,6 +418,7 @@ duplicate_user_talk_head = { 'commons': None, 'en' : None, + 'ga' : None, 'it' : u'\n\n== File doppio ==\n', 'ja' :None, 'ta' :None, @@ -405,6 +428,7 @@ duplicates_user_talk_text = { 'commons': u'{{subst:User:Filnik/duplicates|File:%s|File:%s}}', # FIXME: it doesn't exist 'en' : None, + 'ga' : None, 'it' : u"{{subst:Progetto:Coordinamento/Immagini/Bot/Messaggi/Duplicati|%s|%s|__botnick__}} --~~~~", 'ja' :None, 'ta' :None, @@ -414,6 +438,7 @@ duplicates_comment_talk = { 'commons': u'Bot: Dupe file found', 'en' : None, + 'ga' : None, 'it' : u"Bot: Notifico il file doppio trovato", 'ja' :None, 'ta' :None, @@ -423,6 +448,7 @@ duplicates_comment_image = { 'commons': u'Bot: Tagging dupe file', 'en' : None, + 'ga' : None, 'it' : u'Bot: File doppio, da cancellare', 'ja' :None, 'ta' :None, @@ -432,6 +458,7 @@ duplicatesRegex = { 'commons': r'\{\{(?:[Tt]emplate:|)[Dd]upe[|}]', 'en' : None, + 'ga' : None, 'it' : r'\{\{(?:[Tt]emplate:|)[Pp]rogetto:[Cc]oordinamento/Immagini/Bot/Template duplicati[|}]', 'ja' :None, 'ta' :None, @@ -441,6 +468,7 @@ category_with_licenses = { 'commons': 'Category:License tags', 'en' : 'Category:Wikipedia image copyright templates', + 'ga' : 'Catagóir:Clibeanna cóipchirt d\'íomhánna', 'it' : 'Categoria:Template Licenze copyright', 'ja' : 'Category:画像の著作権表示テンプレート', 'ko' : '분류:그림 저작권 틀', @@ -454,6 +482,7 @@ emailPageWithText = { 'de':'Benutzer:ABF/D3', 'en':None, + 'ga':None, 'ja':None, 'ta':None, 'zh':None, @@ -462,6 +491,7 @@ emailSubject = { 'de':'Problemen mit Deinem Bild auf der Deutschen Wikipedia', 'en':None, + 'ga':None, 'ja':None, 'ta':None, 'zh':None, @@ -476,7 +506,7 @@ } # Add your project (in alphabetical order) if you want that the bot start -project_inserted = [u'ar', u'commons', u'de', u'en', u'hu', u'it', u'ja', u'ko', u'ta', u'zh'] +project_inserted = [u'ar', u'commons', u'de', u'en', u'ga', u'hu', u'it', u'ja', u'ko', u'ta', u'zh'] # Ok, that's all. What is below, is the rest of code, now the code is fixed and it will run correctly in your project. #########################################################################################################################

1 0

← Newer
1
...
7
8
9
10
11
12
13
...
24
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn October 2009