Pywikipedia-svn

pywikipedia-svn@lists.wikimedia.org

5163 discussions

SVN: [7121] trunk/pywikipedia/welcome.py
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7121 Author: alexsh Date: 2009-08-06 18:10:15 +0000 (Thu, 06 Aug 2009) Log Message: ----------- welcome.py.blocked(): make simple to return user is blocked Modified Paths: -------------- trunk/pywikipedia/welcome.py Modified: trunk/pywikipedia/welcome.py =================================================================== --- trunk/pywikipedia/welcome.py 2009-08-06 17:54:49 UTC (rev 7120) +++ trunk/pywikipedia/welcome.py 2009-08-06 18:10:15 UTC (rev 7121) @@ -497,22 +497,16 @@ #action=query&list=users&ususers=Filnik&usprop=blockinfo """ Function that detects if a user is currently blocked or not. - """ - + """ params = { - 'action' :'query', - 'list' :'users', - 'ususers' :username, - 'usprop' :'blockinfo', - } + 'action': 'query', + 'list': 'users', + 'ususers': username, + 'usprop': 'blockinfo', + } - data = query.GetData(params, encodeTitle = False) # If there's not the blockedby parameter (that means the user isn't blocked), it will return False otherwise True. - try: - blockedBy = data['query']['users'][0]['blockedby'] - except KeyError: - return False # No he's not - return True # Yes is blocked + return query.GetData(params, encodeTitle = False)['query']['users'][0].has_key('blockedby') def defineSign(wsite, signPageTitle, fileSignName = None, fileOption = False): """ Function to load the random signatures. """

1 0

SVN: [7120] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7120 Author: alexsh Date: 2009-08-06 17:54:49 +0000 (Thu, 06 Aug 2009) Log Message: ----------- patch bug #2832749 #2820797 Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py trunk/pywikipedia/featured.py trunk/pywikipedia/lonelypages.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2009-08-06 13:55:25 UTC (rev 7119) +++ trunk/pywikipedia/families/wikipedia_family.py 2009-08-06 17:54:49 UTC (rev 7120) @@ -652,6 +652,7 @@ 'an': [u'Desambig', u'Disambig'], 'ar': [u'Disambig', u'توضيح'], 'arc': [u'ܕ'], + 'arz': [u'توضيح'], 'ast': [u'Dixebra'], 'av': [u'Неоднозначность'], 'az': [u'Dəqiqləşdirmə'], Modified: trunk/pywikipedia/featured.py =================================================================== --- trunk/pywikipedia/featured.py 2009-08-06 13:55:25 UTC (rev 7119) +++ trunk/pywikipedia/featured.py 2009-08-06 17:54:49 UTC (rev 7120) @@ -175,6 +175,7 @@ 'ko': (CAT, u"알찬 글"), 'ksh': (CAT, u"Exzälenter Aatikkel"), 'la': (CAT, u"Paginae mensis"), + 'li': (CAT, u"Wikipedia:Sjterartikele"), 'lmo': (CAT, u"Articol ben faa"), 'lo': (CAT, u"ບົດຄວາມດີເດັ່ນ"), 'lt': (CAT, u"Vikipedijos pavyzdiniai straipsniai"), Modified: trunk/pywikipedia/lonelypages.py =================================================================== --- trunk/pywikipedia/lonelypages.py 2009-08-06 13:55:25 UTC (rev 7119) +++ trunk/pywikipedia/lonelypages.py 2009-08-06 17:54:49 UTC (rev 7120) @@ -66,6 +66,7 @@ 'ar':u'{{يتيمة|تاريخ={{نسخ:اسم_شهر}} {{نسخ:عام}}}}', 'en':u'{{Orphan|date={{subst:CURRENTMONTHNAME}} {{subst:CURRENTYEAR}}}}', 'it':u'{{O||mese={{subst:CURRENTMONTHNAME}} {{subst:CURRENTYEAR}}}}', + 'ja':u'{{孤立|{{subst:DATE}}}}', 'zh':u'{{subst:Orphan/auto}}', } @@ -74,6 +75,7 @@ 'ar':u'بوت: صفحة يتيمة، إضافة قالب', 'en':u'Bot: Orphan page, add template', 'it':u'Bot: Voce orfana, aggiungo template {{O}}', + 'ja':u'ロボットによる: 孤立したページへのテンプレート貼付け', 'zh':u'機器人: 本頁的鏈入頁面太少', } @@ -83,6 +85,7 @@ 'ar':u'بوت: إضافة صفحة توضيح', 'en':u'Bot: Adding a disambig page', 'it':u'Bot: Aggiungo una disambigua', + 'ja':u'ロボットによる: 曖昧さ回避の追加', 'zh':u'機器人: 增加消歧義頁面', } @@ -93,6 +96,7 @@ 'ar': [ur'\{\{(?:قالب:|)(يتيمة)[\|\}]'], 'en': [r'\{\{(?:template:|)(orphan)[\|\}]', r'\{\{(?:template:|)(wi)[\|\}]'], 'it': [r'\{\{(?:template:|)(o)[\|\}]'], + 'ja': [ur'\{\{(?:template:|)(孤立)[\|\}]'], 'zh': [r'\{\{(?:template:|)(orphan)[\|\}]'], }

1 0

SVN: [7119] trunk/pywikipedia/casechecker.py
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7119 Author: alexsh Date: 2009-08-06 13:55:25 +0000 (Thu, 06 Aug 2009) Log Message: ----------- Fix:query.getData() parameter Modified Paths: -------------- trunk/pywikipedia/casechecker.py Modified: trunk/pywikipedia/casechecker.py =================================================================== --- trunk/pywikipedia/casechecker.py 2009-08-06 11:08:52 UTC (rev 7118) +++ trunk/pywikipedia/casechecker.py 2009-08-06 13:55:25 UTC (rev 7119) @@ -232,7 +232,7 @@ while True: # Get data self.params['gapfrom'] = self.apfrom - data = query.GetData(self.site.lang, self.params, wikipedia.verbose, True) + data = query.GetData(self.params, self.site, wikipedia.verbose, True) try: self.apfrom = data['query-continue']['allpages']['gapfrom'] except:

1 0

SVN: [7118] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7118 Author: alexsh Date: 2009-08-06 11:08:52 +0000 (Thu, 06 Aug 2009) Log Message: ----------- wikipedia.py: Put API version lock. according [[mw:API:Query]] information (some API query is not necessary because they start from 1.9) Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-06 10:45:42 UTC (rev 7117) +++ trunk/pywikipedia/wikipedia.py 2009-08-06 11:08:52 UTC (rev 7118) @@ -5133,7 +5133,7 @@ retry_idle_time = 1 while True: - if config.use_api: + if config.use_api and self.versionnumber() >= 12: params = { 'action':'query', 'meta':'allmessages', @@ -5718,7 +5718,7 @@ break def randompage(self): - if config.use_api: + if config.use_api and self.versionnumber() >= 12: params = { 'action': 'query', 'list': 'random', @@ -5736,7 +5736,7 @@ return Page(self, m.group('title')) def randomredirectpage(self): - if config.use_api: + if config.use_api and self.versionnumber() >= 12: params = { 'action': 'query', 'list': 'random', @@ -5980,7 +5980,7 @@ urlsToRetrieve = [siteurl] if not siteurl.startswith('*.'): urlsToRetrieve.append('*.' + siteurl) - if config.use_api: + if config.use_api and self.versionnumber() >= 11: output(u'Querying API exturlusage...') for url in urlsToRetrieve: params = { @@ -6593,6 +6593,9 @@ NOTE 2: it returns the image WITHOUT the image namespace. """ + if self.versionnumber() < 12: + return None + if hash_found is None: # If the hash is none return None and not continue return None # Now get all the images with the same hash

1 0

SVN: [7117] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7117 Author: alexsh Date: 2009-08-06 10:45:42 +0000 (Thu, 06 Aug 2009) Log Message: ----------- *Site()._getUserData(): add description for collect data from API. *Page().delete(): Add API option to delete page.(tested on redirect.py broken, could reduce more time to load delete page.) Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-06 07:15:41 UTC (rev 7116) +++ trunk/pywikipedia/wikipedia.py 2009-08-06 10:45:42 UTC (rev 7117) @@ -2396,50 +2396,71 @@ answer = 'y' self.site()._noDeletePrompt = True if answer == 'y': - host = self.site().hostname() - address = self.site().delete_address(self.urlname()) - - reason = reason.encode(self.site().encoding()) + token = self.site().getToken(self, sysop = True) - predata = { - 'wpDeleteReasonList': 'other', - 'wpReason': reason, - 'wpComment': reason, - 'wpConfirm': '1', - 'wpConfirmB': '1' - } - if token: - predata['wpEditToken'] = token - if self.site().hostname() in config.authenticate.keys(): - predata['Content-type'] = 'application/x-www-form-urlencoded' - predata['User-agent'] = useragent - data = self.site().urlEncode(predata) - response = urllib2.urlopen(urllib2.Request(self.site().protocol() + '://' + self.site().hostname() + address, data)) - data = u'' - else: - response, data = self.site().postForm(address, predata, sysop = True) - if data: - self.site().checkBlocks(sysop = True) - if self.site().mediawiki_message('actioncomplete') in data: + + if config.use_api and self.site().versionnumber() >= 12: + params = { + 'action': 'delete', + 'title': self.title(), + 'token': token, + 'reason': reason, + } + datas = query.GetData(params, self.site(), sysop = True) + if datas.has_key('delete'): output(u'Page %s deleted' % self.aslink(forceInterwiki = True)) return True - elif self.site().mediawiki_message('cannotdelete') in data: - output(u'Page %s could not be deleted - it doesn\'t exist' % self.aslink(forceInterwiki = True)) - return False else: - output(u'Deletion of %s failed for an unknown reason. The response text is:' % self.aslink(forceInterwiki = True)) - try: - ibegin = data.index('') + 22 - iend = data.index('') - except ValueError: - # if begin/end markers weren't found, show entire HTML file - output(data) + if datas['error']['code'] == 'missingtitle': + output(u'Page %s could not be deleted - it doesn\'t exist' % self.aslink(forceInterwiki = True)) + return False else: - # otherwise, remove the irrelevant sections - data = data[ibegin:iend] - output(data) - return False + output(u'Deletion of %s failed for an unknown reason. The response text is:' % self.aslink(forceInterwiki = True)) + output('%s' % datas) + else: + host = self.site().hostname() + address = self.site().delete_address(self.urlname()) + reason = reason.encode(self.site().encoding()) + predata = { + 'wpDeleteReasonList': 'other', + 'wpReason': reason, + 'wpComment': reason, + 'wpConfirm': '1', + 'wpConfirmB': '1' + } + if token: + predata['wpEditToken'] = token + if self.site().hostname() in config.authenticate.keys(): + predata['Content-type'] = 'application/x-www-form-urlencoded' + predata['User-agent'] = useragent + data = self.site().urlEncode(predata) + response = urllib2.urlopen(urllib2.Request(self.site().protocol() + '://' + self.site().hostname() + address, data)) + data = u'' + else: + response, data = self.site().postForm(address, predata, sysop = True) + if data: + self.site().checkBlocks(sysop = True) + if self.site().mediawiki_message('actioncomplete') in data: + output(u'Page %s deleted' % self.aslink(forceInterwiki = True)) + return True + elif self.site().mediawiki_message('cannotdelete') in data: + output(u'Page %s could not be deleted - it doesn\'t exist' % self.aslink(forceInterwiki = True)) + return False + else: + output(u'Deletion of %s failed for an unknown reason. The response text is:' % self.aslink(forceInterwiki = True)) + try: + ibegin = data.index('') + 22 + iend = data.index('') + except ValueError: + # if begin/end markers weren't found, show entire HTML file + output(data) + else: + # otherwise, remove the irrelevant sections + data = data[ibegin:iend] + output(data) + return False + def loadDeletedRevisions(self): """Retrieve all deleted revisions for this Page from Special/Undelete. @@ -4879,8 +4900,7 @@ index = self._userIndex(sysop) if type(text) == dict: #text is dict, query from API - # Check for blocks - but only if version is 1.11 (userinfo is available) - # and the user data was not yet loaded + # Check for blocks if text.has_key('blockedby') and not self._isBlocked[index]: # Write a warning if not shown earlier if sysop: @@ -4890,7 +4910,7 @@ output(u'WARNING: %s on %s is blocked. Editing using this account will stop the run.' % (account, self)) self._isBlocked[index] = text.has_key('blockedby') - # Check for new messages, show key 'messages' in dict. + # Check for new messages, the data must had key 'messages' in dict. if text.has_key('messages'): if not self._messages[index]: # User has *new* messages @@ -4907,16 +4927,16 @@ return # Get username. - # anonymous mode will show key 'anon' + # The data in anonymous mode had key 'anon' + # if 'anon' exist, username is IP address, not to collect it right now if not text.has_key('anon'): self._isLoggedIn[index] = True self._userName[index] = text['name'] else: self._isLoggedIn[index] = False - # No idea what is the user name, and it isn't important self._userName[index] = None - # Check user groups and rights + # Get user groups and rights if text.has_key('groups') and text['groups'] != []: self._rights[index] = text['groups'] self._rights[index].extend(text['rights']) @@ -4933,7 +4953,7 @@ if sysop and 'sysop' not in self._rights[index]: output(u'WARNING: Your sysop account on %s does not seem to have sysop rights. You may not be able to perform any sysop-restricted actions using it.' % self) else: - # key groups is not exists, setup a default rights + # 'groups' is not exists, set default rights self._rights[index] = [] if self._isLoggedIn[index]: # Logged in user @@ -4948,13 +4968,13 @@ #remove Duplicate rights self._rights[index] = list(set(self._rights[index])) - # Search for a token + # Get token if text.has_key('preferencestoken') and len(text['preferencestoken']) > 2: # anonymous token is '+\\', check len('+\\') = 2 # if preferencestoken > 2, it must be loggedin. self._token[index] = text['preferencestoken'] if self._rights[index] is not None: - # In this case, token and rights are loaded - user data is now loaded + # Token and rights are loaded - user data is now loaded self._userData[index] = True else: output(u'WARNING: Token not found on %s. You will not be able to edit any page.' % self) @@ -5213,6 +5233,7 @@ config.use_api = False # Get data + # API Userinfo is available from version 1.11 if config.use_api and self.versionnumber() >= 11: #Query userinfo params = {

1 0

SVN: [7116] trunk/pywikipedia/query.py
by alexsh＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7116 Author: alexsh Date: 2009-08-06 07:15:41 +0000 (Thu, 06 Aug 2009) Log Message: ----------- query.GetData(): Fix postData cookie cannot use sysop data Modified Paths: -------------- trunk/pywikipedia/query.py Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2009-08-06 06:00:28 UTC (rev 7115) +++ trunk/pywikipedia/query.py 2009-08-06 07:15:41 UTC (rev 7116) @@ -81,7 +81,7 @@ try: jsontext = "Nothing received" if params['action'] in postAC: - res, jsontext = site.postData(path, urllib.urlencode(params.items()), cookies=site.cookies(), sysop=sysop) + res, jsontext = site.postData(path, urllib.urlencode(params.items()), cookies=site.cookies(sysop=sysop), sysop=sysop) else: if back_response: res, jsontext = site.getUrl( path, retry=True, data=data, sysop=sysop, back_response=True)

1 0

SVN: [7115] trunk/pywikipedia/replace.py
by purodha＠svn.wikimedia.org 06 Aug '09

06 Aug '09

Revision: 7115 Author: purodha Date: 2009-08-06 06:00:28 +0000 (Thu, 06 Aug 2009) Log Message: ----------- Add -query:(pagecount) parameter to replace.py - same as interwiki.py has. Modified Paths: -------------- trunk/pywikipedia/replace.py Modified: trunk/pywikipedia/replace.py =================================================================== --- trunk/pywikipedia/replace.py 2009-08-05 21:40:30 UTC (rev 7114) +++ trunk/pywikipedia/replace.py 2009-08-06 06:00:28 UTC (rev 7115) @@ -62,6 +62,9 @@ the bot will check every regex without waiting using all the resources. This will slow it down between a regex and another in order not to waste too much CPU. + +-query: The maximum number of pages that the bot will load at once. + Default value is 60. Ignored when reading an XML file. -fix:XYZ Perform one of the predefined replacements tasks, which are given in the dictionary 'fixes' defined inside the file @@ -495,6 +498,8 @@ allowoverlap = False # Do not recurse replacement recursive = False + # This is the maximum number of pages to load per query + maxquerysize = 60 # This factory is responsible for processing command line arguments # that are also used by other scripts and that determine on which pages # to work on. @@ -561,6 +566,8 @@ summary_commandline = True elif arg.startswith('-allowoverlap'): allowoverlap = True + elif arg.startswith('-query:'): + maxquerysize = int(arg[7:]) else: if not genFactory.handleArg(arg): commandline_replacements.append(arg) @@ -696,7 +703,7 @@ preloadingGen = pagegenerators.PreloadingGenerator(gen, pageNumber=20, lookahead=100) else: - preloadingGen = pagegenerators.PreloadingGenerator(gen, pageNumber=60) + preloadingGen = pagegenerators.PreloadingGenerator(gen, pageNumber=maxquerysize) bot = ReplaceRobot(preloadingGen, replacements, exceptions, acceptall, allowoverlap, recursive, add_cat, sleep, editSummary) bot.run()

1 0

SVN: [7114] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 05 Aug '09

05 Aug '09

Revision: 7114 Author: alexsh Date: 2009-08-05 21:40:30 +0000 (Wed, 05 Aug 2009) Log Message: ----------- Remove all 'useAPI=True' tag for function set useAPI default True Modified Paths: -------------- trunk/pywikipedia/casechecker.py trunk/pywikipedia/catlib.py trunk/pywikipedia/imageuncat.py trunk/pywikipedia/maintenance/family_check.py trunk/pywikipedia/welcome.py trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/casechecker.py =================================================================== --- trunk/pywikipedia/casechecker.py 2009-08-05 21:39:21 UTC (rev 7113) +++ trunk/pywikipedia/casechecker.py 2009-08-05 21:40:30 UTC (rev 7114) @@ -200,14 +200,14 @@ wlpage = self.whitelists[self.site.lang] wikipedia.output(u'Loading whitelist from %s' % wlpage) wlparams = { - 'action' : 'query', - 'prop' : 'links', - 'titles' : wlpage, - 'redirects' : '', - 'indexpageids' : '', - } + 'action' : 'query', + 'prop' : 'links', + 'titles' : wlpage, + 'redirects' : '', + 'indexpageids' : '', + } - data = query.GetData(self.site.lang, wlparams, wikipedia.verbose, useAPI=True, encodeTitle=False) + data = query.GetData(wlparams, wikipedia.getSite(self.site.lang), wikipedia.verbose, encodeTitle=False) if len(data['query']['pageids']) == 1: pageid = data['query']['pageids'][0] links = data['query']['pages'][pageid]['links'] Modified: trunk/pywikipedia/catlib.py =================================================================== --- trunk/pywikipedia/catlib.py 2009-08-05 21:39:21 UTC (rev 7113) +++ trunk/pywikipedia/catlib.py 2009-08-05 21:40:30 UTC (rev 7114) @@ -533,8 +533,7 @@ 'cmtitle' :CatName, } - data = query.GetData(params, site = site, - useAPI = True, encodeTitle = False) + data = query.GetData(params, site, encodeTitle = False) categories_parsed.append(CatName) try: members = data['query']['categorymembers'] Modified: trunk/pywikipedia/imageuncat.py =================================================================== --- trunk/pywikipedia/imageuncat.py 2009-08-05 21:39:21 UTC (rev 7113) +++ trunk/pywikipedia/imageuncat.py 2009-08-05 21:40:30 UTC (rev 7114) @@ -1243,7 +1243,7 @@ 'leend' :today.strftime(dateformat) } - data = query.GetData(params, site, useAPI = True, encodeTitle = False) + data = query.GetData(params, site, encodeTitle = False) try: for item in data['query']['logevents']: result.append(item['title']) @@ -1279,7 +1279,7 @@ 'rctype' :'edit|log', } - data = query.GetData(params, site, useAPI = True, encodeTitle = False) + data = query.GetData(params, site, encodeTitle = False) try: for item in data['query']['recentchanges']: result.append(item['title']) Modified: trunk/pywikipedia/maintenance/family_check.py =================================================================== --- trunk/pywikipedia/maintenance/family_check.py 2009-08-05 21:39:21 UTC (rev 7113) +++ trunk/pywikipedia/maintenance/family_check.py 2009-08-05 21:40:30 UTC (rev 7114) @@ -17,7 +17,7 @@ 'meta': 'siteinfo', 'siprop': 'namespaces'} try: - data = query.GetData(predata, site = site, useAPI = True)['query']['namespaces'] + data = query.GetData(predata, site)['query']['namespaces'] except wikipedia.ServerError, e: wikipedia.output(u'Warning! %s: %s' % (site, e)) return Modified: trunk/pywikipedia/welcome.py =================================================================== --- trunk/pywikipedia/welcome.py 2009-08-05 21:39:21 UTC (rev 7113) +++ trunk/pywikipedia/welcome.py 2009-08-05 21:40:30 UTC (rev 7114) @@ -506,8 +506,7 @@ 'usprop' :'blockinfo', } - data = query.GetData(params, - useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) # If there's not the blockedby parameter (that means the user isn't blocked), it will return False otherwise True. try: blockedBy = data['query']['users'][0]['blockedby'] Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-05 21:39:21 UTC (rev 7113) +++ trunk/pywikipedia/wikipedia.py 2009-08-05 21:40:30 UTC (rev 7114) @@ -919,8 +919,7 @@ 'prop' :'info', 'titles' :self.title(), } - data = query.GetData(params, - useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) pageid = data['query']['pages'].keys()[0] if data['query']['pages'][pageid].keys()[0] == 'lastrevid': return data['query']['pages'][pageid]['lastrevid'] # if ok, @@ -952,8 +951,7 @@ 'tllimit' :tllimit, } - data = query.GetData(params, - useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) try: pageid = data['query']['pages'].keys()[0] except KeyError: @@ -1327,7 +1325,7 @@ #if titles: # predata['titles'] = query.ListToParam(titles) - text = query.GetData(predata, useAPI = True)['query']['pages'] + text = query.GetData(predata)['query']['pages'] for pageid in text: if text[pageid].has_key('missing'): @@ -2765,7 +2763,7 @@ 'titles' :self.title(), } try: - data = query.GetData(params, useAPI = True, encodeTitle = False)['query']['pages'] + data = query.GetData(params, encodeTitle = False)['query']['pages'] except KeyError: raise NoPage(u'API Error, nothing found in the APIs') @@ -2829,7 +2827,7 @@ 'titles' :self.title(), 'iiprop' :'url', } - imagedata = query.GetData(params, useAPI = True, encodeTitle = False) + imagedata = query.GetData(params, encodeTitle = False) try: url=imagedata['query']['pages'].values()[0]['imageinfo'][0]['url'] # urlR = re.compile(r'<div class="fullImageLink" id="file">.*?<a href="(?P<url>[^ ]+?)"(?! class="image")|<span class="dangerousLink"><a href="(?P<url2>.+?)"', re.DOTALL) @@ -2899,7 +2897,7 @@ 'prop' :'imageinfo', 'titles' :self.title(), } - data = query.GetData(params, useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) try: # We don't know the page's id, if any other better idea please change it pageid = data['query']['pages'].keys()[0] @@ -2921,7 +2919,7 @@ 'iiprop' :'sha1', } # First of all we need the Hash that identify an image - data = query.GetData(params, useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) pageid = data['query']['pages'].keys()[0] try: hash_found = data['query']['pages'][pageid][u'imageinfo'][0][u'sha1'] @@ -4505,7 +4503,7 @@ 'meta': 'userinfo', 'uiprop': 'blockinfo', } - data = query.GetData(params, self, useAPI = True)['query']['userinfo'] + data = query.GetData(params, self)['query']['userinfo'] return data.has_key('blockby') except NotImplementedError: return False @@ -5121,7 +5119,7 @@ 'meta':'allmessages', } try: - datas = query.GetData(params, useAPI = True)['query']['allmessages'] + datas = query.GetData(params)['query']['allmessages'] except KeyError: raise ServerError("The APIs don't return data, the site may be down") except NotImplementedError: @@ -5222,7 +5220,7 @@ 'meta': 'userinfo', 'uiprop': 'blockinfo|groups|rights|hasmsg|ratelimits|preferencestoken', } - text = query.GetData(params, site = self, useAPI = True, sysop=sysop)['query']['userinfo'] + text = query.GetData(params, self, sysop=sysop)['query']['userinfo'] ##output('%s' % text) # for debug use only else: url = self.edit_address('Non-existing_page') @@ -5503,8 +5501,7 @@ if leuser is not None: params['leuser'] = leuser if letitle is not None: params['letitle'] = letitle while True: - data = query.GetData(params, - useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) try: imagesData = data['query']['logevents'] except KeyError: @@ -5572,8 +5569,7 @@ if rcshow is not None: params['rcshow'] = rcshow if rctype is not None: params['rctype'] = rctype while True: - data = query.GetData(params, - useAPI = True, encodeTitle = False) + data = query.GetData(params, encodeTitle = False) try: rcData = data['query']['recentchanges'] except KeyError: @@ -5709,7 +5705,7 @@ 'rnlimit': '1', #'': '', } - data = query.GetData(params, useAPI = True) + data = query.GetData(params) return Page(self, data['query']['random'][0]['title']) else: """Yield random page via Special:Random""" @@ -5727,7 +5723,7 @@ 'rnlimit': '1', 'rnredirect': '1', } - data = query.GetData(params, useAPI = True) + data = query.GetData(params) return Page(self, data['query']['random'][0]['title']) else: """Yield random redirect page via Special:RandomRedirect.""" @@ -5781,7 +5777,7 @@ params['apfrom'] = start if throttle: get_throttle() - data = query.GetData(params, useAPI = True) + data = query.GetData(params) for p in data['query']['allpages']: yield Page(self, p['title']) @@ -5973,7 +5969,7 @@ 'euquery': url, } while True: - data = query.GetData(params, useAPI = True) + data = query.GetData(params) if data['query']['exturlusage'] == []: break for pages in data['query']['exturlusage']: @@ -6586,7 +6582,7 @@ 'list' :'allimages', 'aisha1' :hash_found, } - allimages = query.GetData(params, site = getSite(self.lang, self.family), useAPI = True, encodeTitle = False)['query']['allimages'] + allimages = query.GetData(params, getSite(self.lang, self.family), encodeTitle = False)['query']['allimages'] files = list() for imagedata in allimages: image = imagedata[u'name']

1 0

SVN: [7113] trunk/pywikipedia/query.py
by alexsh＠svn.wikimedia.org 05 Aug '09

05 Aug '09

Revision: 7113 Author: alexsh Date: 2009-08-05 21:39:21 +0000 (Wed, 05 Aug 2009) Log Message: ----------- query.py: set useAPI default True..(query.php is removed for long times, not necessary to set it default) Modified Paths: -------------- trunk/pywikipedia/query.py Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2009-08-05 21:25:39 UTC (rev 7112) +++ trunk/pywikipedia/query.py 2009-08-05 21:39:21 UTC (rev 7113) @@ -14,8 +14,7 @@ 'rvprop' :'user|timestamp|content', } - print query.GetData(params, - useAPI = True, encodeTitle = False) + print query.GetData(params, encodeTitle = False) """ # @@ -28,7 +27,7 @@ import wikipedia, simplejson, urllib, time -def GetData(params, site = None, verbose = False, useAPI = False, retryCount = 5, encodeTitle = True, sysop = False, back_response = False): +def GetData(params, site = None, verbose = False, useAPI = True, retryCount = 5, encodeTitle = True, sysop = False, back_response = False): """Get data from the query api, and convert it into a data object """ if site is None:

1 0

SVN: [7112] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 05 Aug '09

05 Aug '09

Revision: 7112 Author: alexsh Date: 2009-08-05 21:25:39 +0000 (Wed, 05 Aug 2009) Log Message: ----------- Site()._load() and Site()._getUserData(): for reduce load time, Add query API option to collect user data. (use ac=query,meta=userinfo) Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-08-05 06:24:39 UTC (rev 7111) +++ trunk/pywikipedia/wikipedia.py 2009-08-05 21:25:39 UTC (rev 7112) @@ -1499,6 +1499,7 @@ predata['wpWatchthis'] = '1' # Give the token, but only if one is supplied. if token: + ##output(token) # for debug use only predata['wpEditToken'] = token # Sorry, single-site exception... @@ -1630,6 +1631,7 @@ except NoUsername: raise LockedPage() if not newToken and "<textarea" in data: + ##if "<textarea" in data: # for debug use only, if badtoken still happen # We might have been using an outdated token output(u"Changing page has failed. Retrying.") return self._putPage(text, comment, watchArticle, minorEdit, newPage, token=self.site().getToken(sysop = sysop, getagain = True), newToken = True, sysop = sysop) @@ -4519,6 +4521,7 @@ else: self._load(sysop = sysop) index = self._userIndex(sysop) + ##output('%s' % self._rights[index]) #for debug use return right in self._rights[index] def server_time(self): @@ -4874,126 +4877,209 @@ * text - the page text * sysop - is the user a sysop? """ - if '<div id="globalWrapper">' not in text: - # Not a wiki page - return index = self._userIndex(sysop) - # Check for blocks - but only if version is 1.11 (userinfo is available) - # and the user data was not yet loaded - if self.versionnumber() >= 11 and (not self._userData[index] or force): - blocked = self._getBlock(sysop = sysop) - if blocked and not self._isBlocked[index]: + if type(text) == dict: #text is dict, query from API + # Check for blocks - but only if version is 1.11 (userinfo is available) + # and the user data was not yet loaded + if text.has_key('blockedby') and not self._isBlocked[index]: # Write a warning if not shown earlier if sysop: account = 'Your sysop account' else: account = 'Your account' output(u'WARNING: %s on %s is blocked. Editing using this account will stop the run.' % (account, self)) - self._isBlocked[index] = blocked + self._isBlocked[index] = text.has_key('blockedby') - # Check for new messages - if '<div class="usermessage">' in text: - if not self._messages[index]: - # User has *new* messages - if sysop: - output(u'NOTE: You have new messages in your sysop account on %s' % self) - else: - output(u'NOTE: You have new messages on %s' % self) - self._messages[index] = True - else: - self._messages[index] = False + # Check for new messages, show key 'messages' in dict. + if text.has_key('messages'): + if not self._messages[index]: + # User has *new* messages + if sysop: + output(u'NOTE: You have new messages in your sysop account on %s' % self) + else: + output(u'NOTE: You have new messages on %s' % self) + self._messages[index] = True + else: + self._messages[index] = False - # Don't perform other checks if the data was already loaded - if self._userData[index] and not force: - return + # Don't perform other checks if the data was already loaded + if self._userData[index] and not force: + return - # Search for the the user page link at the top. - # Note that the link of anonymous users (which doesn't exist at all - # in Wikimedia sites) has the ID pt-anonuserpage, and thus won't be - # found here. - userpageR = re.compile('<li id="pt-userpage"><a href=".+?">(?P<username>.+?)</a></li>') - m = userpageR.search(text) - if m: - self._isLoggedIn[index] = True - self._userName[index] = m.group('username') - else: - self._isLoggedIn[index] = False - # No idea what is the user name, and it isn't important - self._userName[index] = None + # Get username. + # anonymous mode will show key 'anon' + if not text.has_key('anon'): + self._isLoggedIn[index] = True + self._userName[index] = text['name'] + else: + self._isLoggedIn[index] = False + # No idea what is the user name, and it isn't important + self._userName[index] = None - # Check user groups, if possible (introduced in 1.10) - groupsR = re.compile(r'var wgUserGroups = \[\"(.+)\"\];') - m = groupsR.search(text) - checkLocal = True - if default_code in self.family.cross_allowed: # if current languages in cross allowed list, check global bot flag. - globalgroupsR = re.compile(r'var wgGlobalGroups = \[\"(.+)\"\];') - mg = globalgroupsR.search(text) - if mg: # the account had global permission - globalRights = mg.group(1) - globalRights = globalRights.split('","') - self._rights[index] = globalRights + # Check user groups and rights + if text.has_key('groups') and text['groups'] != []: + self._rights[index] = text['groups'] + self._rights[index].extend(text['rights']) + # Warnings + # Don't show warnings for not logged in users, they will just fail to + # do any action if self._isLoggedIn[index]: - if 'Global_bot' in globalRights: # This account has the global bot flag, no need to check local flags. - checkLocal = False - else: - output(u'Your bot account does not have global the bot flag, checking local flag.') + if 'bot' not in self._rights[index] and config.notify_unflagged_bot: + # Sysop + bot flag = Sysop flag in MediaWiki < 1.7.1? + if sysop: + output(u'Note: Your sysop account on %s does not have a bot flag. Its edits will be visible in the recent changes.' % self) + else: + output(u'WARNING: Your account on %s does not have a bot flag. Its edits will be visible in the recent changes and it may get blocked.' % self) + if sysop and 'sysop' not in self._rights[index]: + output(u'WARNING: Your sysop account on %s does not seem to have sysop rights. You may not be able to perform any sysop-restricted actions using it.' % self) + else: + # key groups is not exists, setup a default rights + self._rights[index] = [] + if self._isLoggedIn[index]: + # Logged in user + self._rights[index].append('user') + # Assume bot, and thus autoconfirmed + self._rights[index].extend(['bot', 'autoconfirmed']) + if sysop: + # Assume user reported as a sysop indeed has the sysop rights + self._rights[index].append('sysop') + # Assume the user has the default rights if API not query back + self._rights[index].extend(['read', 'createaccount', 'edit', 'upload', 'createpage', 'createtalk', 'move', 'upload']) + #remove Duplicate rights + self._rights[index] = list(set(self._rights[index])) + + # Search for a token + if text.has_key('preferencestoken') and len(text['preferencestoken']) > 2: + # anonymous token is '+\\', check len('+\\') = 2 + # if preferencestoken > 2, it must be loggedin. + self._token[index] = text['preferencestoken'] + if self._rights[index] is not None: + # In this case, token and rights are loaded - user data is now loaded + self._userData[index] = True + else: + output(u'WARNING: Token not found on %s. You will not be able to edit any page.' % self) else: - if verbose: output(u'Note: this language does not allow global bots.') - if m and checkLocal: - rights = m.group(1) - rights = rights.split('", "') - if '*' in rights: - rights.remove('*') - self._rights[index] = rights - # Warnings - # Don't show warnings for not logged in users, they will just fail to - # do any action - if self._isLoggedIn[index]: - if 'bot' not in self._rights[index] and config.notify_unflagged_bot: - # Sysop + bot flag = Sysop flag in MediaWiki < 1.7.1? + #ordinary mode to get data from edit page HTMLs and JavaScripts + + if '<div id="globalWrapper">' not in text: + # Not a wiki page + return + # Check for blocks - but only if version is 1.11 (userinfo is available) + # and the user data was not yet loaded + if self.versionnumber() >= 11 and (not self._userData[index] or force): + blocked = self._getBlock(sysop = sysop) + if blocked and not self._isBlocked[index]: + # Write a warning if not shown earlier if sysop: - output(u'Note: Your sysop account on %s does not have a bot flag. Its edits will be visible in the recent changes.' % self) + account = 'Your sysop account' else: - output(u'WARNING: Your account on %s does not have a bot flag. Its edits will be visible in the recent changes and it may get blocked.' % self) - if sysop and 'sysop' not in self._rights[index]: - output(u'WARNING: Your sysop account on %s does not seem to have sysop rights. You may not be able to perform any sysop-restricted actions using it.' % self) - else: - # We don't have wgUserGroups, and can't check the rights - self._rights[index] = [] - if self._isLoggedIn[index]: - # Logged in user - self._rights[index].append('user') - # Assume bot, and thus autoconfirmed - self._rights[index].extend(['bot', 'autoconfirmed']) - if sysop: - # Assume user reported as a sysop indeed has the sysop rights - self._rights[index].append('sysop') - # Assume the user has the default rights - self._rights[index].extend(['read', 'createaccount', 'edit', 'upload', 'createpage', 'createtalk', 'move', 'upload']) - if 'bot' in self._rights[index] or 'sysop' in self._rights[index]: - self._rights[index].append('apihighlimits') - if 'sysop' in self._rights[index]: - self._rights[index].extend(['delete', 'undelete', 'block', 'protect', 'import', 'deletedhistory', 'unwatchedpages']) + account = 'Your account' + output(u'WARNING: %s on %s is blocked. Editing using this account will stop the run.' % (account, self)) + self._isBlocked[index] = blocked - # Search for a token - tokenR = re.compile(r"\<input type='hidden' value=\"(.*?)\" name=\"wpEditToken\"") - tokenloc = tokenR.search(text) - if tokenloc: - self._token[index] = tokenloc.group(1) - if self._rights[index] is not None: - # In this case, token and rights are loaded - user data is now loaded - self._userData[index] = True - else: - # Token not found - # Possible reason for this is the user is blocked, don't show a - # warning in this case, otherwise do show a warning - # Another possible reason is that the page cannot be edited - ensure - # there is a textarea and the tab "view source" is not shown - if u'<textarea' in text and u'<li id="ca-viewsource"' not in text and not self._isBlocked[index]: + # Check for new messages + if '<div class="usermessage">' in text: + if not self._messages[index]: + # User has *new* messages + if sysop: + output(u'NOTE: You have new messages in your sysop account on %s' % self) + else: + output(u'NOTE: You have new messages on %s' % self) + self._messages[index] = True + else: + self._messages[index] = False + + # Don't perform other checks if the data was already loaded + if self._userData[index] and not force: + return + + # Search for the the user page link at the top. + # Note that the link of anonymous users (which doesn't exist at all + # in Wikimedia sites) has the ID pt-anonuserpage, and thus won't be + # found here. + userpageR = re.compile('<li id="pt-userpage"><a href=".+?">(?P<username>.+?)</a></li>') + m = userpageR.search(text) + if m: + self._isLoggedIn[index] = True + self._userName[index] = m.group('username') + else: + self._isLoggedIn[index] = False + # No idea what is the user name, and it isn't important + self._userName[index] = None + + # Check user groups, if possible (introduced in 1.10) + groupsR = re.compile(r'var wgUserGroups = \[\"(.+)\"\];') + m = groupsR.search(text) + checkLocal = True + if default_code in self.family.cross_allowed: # if current languages in cross allowed list, check global bot flag. + globalgroupsR = re.compile(r'var wgGlobalGroups = \[\"(.+)\"\];') + mg = globalgroupsR.search(text) + if mg: # the account had global permission + globalRights = mg.group(1) + globalRights = globalRights.split('","') + self._rights[index] = globalRights + if self._isLoggedIn[index]: + if 'Global_bot' in globalRights: # This account has the global bot flag, no need to check local flags. + checkLocal = False + else: + output(u'Your bot account does not have global the bot flag, checking local flag.') + else: + if verbose: output(u'Note: this language does not allow global bots.') + if m and checkLocal: + rights = m.group(1) + rights = rights.split('", "') + if '*' in rights: + rights.remove('*') + self._rights[index] = rights + # Warnings + # Don't show warnings for not logged in users, they will just fail to + # do any action + if self._isLoggedIn[index]: + if 'bot' not in self._rights[index] and config.notify_unflagged_bot: + # Sysop + bot flag = Sysop flag in MediaWiki < 1.7.1? + if sysop: + output(u'Note: Your sysop account on %s does not have a bot flag. Its edits will be visible in the recent changes.' % self) + else: + output(u'WARNING: Your account on %s does not have a bot flag. Its edits will be visible in the recent changes and it may get blocked.' % self) + if sysop and 'sysop' not in self._rights[index]: + output(u'WARNING: Your sysop account on %s does not seem to have sysop rights. You may not be able to perform any sysop-restricted actions using it.' % self) + else: + # We don't have wgUserGroups, and can't check the rights + self._rights[index] = [] + if self._isLoggedIn[index]: + # Logged in user + self._rights[index].append('user') + # Assume bot, and thus autoconfirmed + self._rights[index].extend(['bot', 'autoconfirmed']) + if sysop: + # Assume user reported as a sysop indeed has the sysop rights + self._rights[index].append('sysop') + # Assume the user has the default rights + self._rights[index].extend(['read', 'createaccount', 'edit', 'upload', 'createpage', 'createtalk', 'move', 'upload']) + if 'bot' in self._rights[index] or 'sysop' in self._rights[index]: + self._rights[index].append('apihighlimits') + if 'sysop' in self._rights[index]: + self._rights[index].extend(['delete', 'undelete', 'block', 'protect', 'import', 'deletedhistory', 'unwatchedpages']) + + # Search for a token + tokenR = re.compile(r"\<input type='hidden' value=\"(.*?)\" name=\"wpEditToken\"") + tokenloc = tokenR.search(text) + if tokenloc: + self._token[index] = tokenloc.group(1) + if self._rights[index] is not None: + # In this case, token and rights are loaded - user data is now loaded + self._userData[index] = True + else: # Token not found - output(u'WARNING: Token not found on %s. You will not be able to edit any page.' % self) + # Possible reason for this is the user is blocked, don't show a + # warning in this case, otherwise do show a warning + # Another possible reason is that the page cannot be edited - ensure + # there is a textarea and the tab "view source" is not shown + if u'<textarea' in text and u'<li id="ca-viewsource"' not in text and not self._isBlocked[index]: + # Token not found + output(u'WARNING: Token not found on %s. You will not be able to edit any page.' % self) def mediawiki_message(self, key): """Return the MediaWiki message text for key "key" """ @@ -5121,10 +5207,26 @@ if verbose: output(u'Getting information for site %s' % self) - + + try: + api_url = self.api_address() + del api_url + except NotImplementedError: + config.use_api = False + # Get data - url = self.edit_address('Non-existing_page') - text = self.getUrl(url, sysop = sysop) + if config.use_api and self.versionnumber() >= 11: + #Query userinfo + params = { + 'action': 'query', + 'meta': 'userinfo', + 'uiprop': 'blockinfo|groups|rights|hasmsg|ratelimits|preferencestoken', + } + text = query.GetData(params, site = self, useAPI = True, sysop=sysop)['query']['userinfo'] + ##output('%s' % text) # for debug use only + else: + url = self.edit_address('Non-existing_page') + text = self.getUrl(url, sysop = sysop) # Parse data self._getUserData(text, sysop = sysop, force = force) @@ -5862,7 +5964,7 @@ if not siteurl.startswith('*.'): urlsToRetrieve.append('*.' + siteurl) if config.use_api: - output(u'Querying API...') + output(u'Querying API exturlusage...') for url in urlsToRetrieve: params = { 'action': 'query',

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn