Pywikipedia-svn October 2009

pywikipedia-svn@lists.wikimedia.org

10 participants
237 discussions

by nicdumz＠svn.wikimedia.org

Revision: 7366 Author: nicdumz Date: 2009-10-04 15:28:18 +0000 (Sun, 04 Oct 2009) Log Message: ----------- xmlreader: parse redirect information from xml Original patch from Santiago Mola Modified Paths: -------------- trunk/pywikipedia/tests/test_xmlreader.py trunk/pywikipedia/xmlreader.py Added Paths: ----------- trunk/pywikipedia/tests/data/article-pyrus.xml Added: trunk/pywikipedia/tests/data/article-pyrus.xml =================================================================== --- trunk/pywikipedia/tests/data/article-pyrus.xml (rev 0) +++ trunk/pywikipedia/tests/data/article-pyrus.xml 2009-10-04 15:28:18 UTC (rev 7366) @@ -0,0 +1,101 @@ +<mediawiki xmlns="http://www.mediawiki.org/xml/export-0.3/" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.mediawiki.org/xml/export-0.3/ http://www.mediawiki.org/xml/export-0.3.xsd" version="0.3" xml:lang="en"> + <siteinfo> + <sitename>Wikipedia</sitename> + <base>http://en.wikipedia.org/wiki/Main_Page</base> + <generator>MediaWiki 1.16alpha-wmf</generator> + <case>first-letter</case> + <namespaces> + <namespace key="-2">Media</namespace> + <namespace key="-1">Special</namespace> + <namespace key="0" /> + <namespace key="1">Talk</namespace> + <namespace key="2">User</namespace> + <namespace key="3">User talk</namespace> + <namespace key="4">Wikipedia</namespace> + <namespace key="5">Wikipedia talk</namespace> + <namespace key="6">File</namespace> + <namespace key="7">File talk</namespace> + <namespace key="8">MediaWiki</namespace> + <namespace key="9">MediaWiki talk</namespace> + <namespace key="10">Template</namespace> + <namespace key="11">Template talk</namespace> + <namespace key="12">Help</namespace> + <namespace key="13">Help talk</namespace> + <namespace key="14">Category</namespace> + <namespace key="15">Category talk</namespace> + <namespace key="100">Portal</namespace> + <namespace key="101">Portal talk</namespace> + </namespaces> + </siteinfo> + <page> + <title>Pyrus</title> + <id>9261472</id> + <redirect /> + <revision> + <id>104997415</id> + <timestamp>2007-02-02T02:39:52Z</timestamp> + <contributor> + <username>Melburnian</username> + <id>555187</id> + </contributor> + <comment>moved [[Pyrus]] to [[Pyrus (brand)]]: all links to ''Pyrus'' are related to the pear tree or fruit</comment> + <text xml:space="preserve">#REDIRECT [[Pyrus (brand)]]</text> + </revision> + <revision> + <id>104997738</id> + <timestamp>2007-02-02T02:41:24Z</timestamp> + <contributor> + <username>Melburnian</username> + <id>555187</id> + </contributor> + <comment>all links to ''Pyrus'' are related to the pear tree or fruit</comment> + <text xml:space="preserve">#REDIRECT [[Pear]]</text> + </revision> + <revision> + <id>189729426</id> + <timestamp>2008-02-07T14:06:10Z</timestamp> + <contributor> + <username>Jkokemueller</username> + <id>6303952</id> + </contributor> + <comment>Added disambiguation</comment> + <text xml:space="preserve">'''Pyrus''' may refer to: + +* [[Pear]], trees of the genus ''Pyrus'' and the fruit of that tree, edible in some species +* [[Main//Pyrus DMS]], a [[Document Management System]]</text> + </revision> + <revision> + <id>190346463</id> + <timestamp>2008-02-10T07:21:12Z</timestamp> + <contributor> + <username>IceCreamAntisocial</username> + <id>346507</id> + </contributor> + <minor/> + <comment>rv</comment> + <text xml:space="preserve">#REDIRECT [[Pear]]</text> + </revision> + <revision> + <id>238138507</id> + <timestamp>2008-09-13T12:57:33Z</timestamp> + <contributor> + <username>Cottonapple4</username> + <id>7707615</id> + </contributor> + <comment>[[WP:AES|←]] Redirected page to [[Pear]]</comment> + <text xml:space="preserve">#REDIRECT [[Pear]] +[[Category:Maloideae]]</text> + </revision> + <revision> + <id>238392911</id> + <timestamp>2008-09-14T17:08:56Z</timestamp> + <contributor> + <username>Rkitko</username> + <id>536375</id> + </contributor> + <minor/> + <comment>Reverted edits by [[Special:Contributions/Cottonapple4|Cottonapple4]] ([[User talk:Cottonapple4|talk]]) to last version by IceCreamAntisocial</comment> + <text xml:space="preserve">#REDIRECT [[Pear]]</text> + </revision> + </page> +</mediawiki> Modified: trunk/pywikipedia/tests/test_xmlreader.py =================================================================== --- trunk/pywikipedia/tests/test_xmlreader.py 2009-10-04 15:11:01 UTC (rev 7365) +++ trunk/pywikipedia/tests/test_xmlreader.py 2009-10-04 15:28:18 UTC (rev 7366) @@ -14,6 +14,7 @@ self.assertEquals(u"24278", pages[0].id) self.assertTrue(pages[0].text.startswith('Pears are [[tree]]s of')) self.assertEquals(u"Quercusrobur", pages[1].username) + self.assertEquals(u"Pear", pages[0].title) def test_XmlDumpFirstRev(self): pages = [r for r in xmlreader.XmlDump("data/article-pear.xml").parse()] @@ -22,7 +23,12 @@ self.assertEquals(u"Pear", pages[0].title) self.assertEquals(u"24278", pages[0].id) self.assertTrue(pages[0].text.startswith('Pears are [[tree]]s of')) + self.assertTrue(not pages[0].isredirect) + def test_XmlDumpRedirect(self): + pages = [r for r in xmlreader.XmlDump("data/article-pyrus.xml").parse()] + self.assertTrue(pages[0].isredirect) + def test_MediaWikiXmlHandler(self): handler = xmlreader.MediaWikiXmlHandler() pages = [] @@ -30,9 +36,9 @@ pages.append(page) handler.setCallback(pageDone) xml.sax.parse("data/article-pear.xml", handler) + self.assertEquals(u"Pear", pages[0].title) self.assertEquals(4, len(pages)) self.assertNotEquals("", pages[0].comment) - if __name__ == '__main__': unittest.main() Modified: trunk/pywikipedia/xmlreader.py =================================================================== --- trunk/pywikipedia/xmlreader.py 2009-10-04 15:11:01 UTC (rev 7365) +++ trunk/pywikipedia/xmlreader.py 2009-10-04 15:28:18 UTC (rev 7366) @@ -56,7 +56,7 @@ """ Represents a page. """ - def __init__(self, title, id, text, username, ipedit, timestamp, editRestriction, moveRestriction, revisionid, comment): + def __init__(self, title, id, text, username, ipedit, timestamp, editRestriction, moveRestriction, revisionid, comment, redirect): # TODO: there are more tags we can read. self.title = title self.id = id @@ -68,6 +68,7 @@ self.moveRestriction = moveRestriction self.revisionid = revisionid self.comment = comment + self.isredirect = redirect class XmlHeaderEntry: @@ -94,6 +95,7 @@ self.id = u'' self.revisionid = u'' self.comment = u'' + self.isredirect = False def setCallback(self, callback): self.callback = callback @@ -159,6 +161,8 @@ self.inContributorTag = False elif name == 'restrictions': self.editRestriction, self.moveRestriction = parseRestrictions(self.restrictions) + elif name == 'redirect': + self.isredirect = True elif name == 'revision': # All done for this. # Remove trailing newlines and spaces @@ -178,7 +182,7 @@ text, self.username, self.ipedit, timestamp, self.editRestriction, self.moveRestriction, - self.revisionid, self.comment) + self.revisionid, self.comment, self.isredirect) self.inRevisionTag = False self.callback(entry) elif self.headercallback: @@ -313,6 +317,7 @@ self.title = elem.findtext("{%s}title" % self.uri) self.pageid = elem.findtext("{%s}id" % self.uri) self.restrictions = elem.findtext("{%s}restrictions" % self.uri) + self.isredirect = elem.findtext("{%s}redirect" % self.uri) is not None def _create_revision(self, revision): """Creates a Single revision""" @@ -332,7 +337,8 @@ editRestriction=editRestriction, moveRestriction=moveRestriction, revisionid=revisionid, - comment=comment + comment=comment, + redirect=self.isredirect ) def regex_parse(self):

14 years, 7 months

SVN: [7365] trunk/pywikipedia/wikipedia.py

by nicdumz＠svn.wikimedia.org

Revision: 7365 Author: nicdumz Date: 2009-10-04 15:11:01 +0000 (Sun, 04 Oct 2009) Log Message: ----------- doc: fixing url Patch by Santiago Mola Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-04 15:08:53 UTC (rev 7364) +++ trunk/pywikipedia/wikipedia.py 2009-10-04 15:11:01 UTC (rev 7365) @@ -7258,7 +7258,7 @@ The differences are highlighted (only on Unix systems) to show which changes were made. """ - # For information on difflib, see http://pydoc.org/2.3/difflib.html + # For information on difflib, see http://docs.python.org/library/difflib.html color = { '+': 'lightgreen', '-': 'lightred',

14 years, 7 months

SVN: [7364] trunk/pywikipedia/replace.py

by nicdumz＠svn.wikimedia.org

Revision: 7364 Author: nicdumz Date: 2009-10-04 15:08:53 +0000 (Sun, 04 Oct 2009) Log Message: ----------- Improving docstrings for multiple replacements. Patch By Santiago Mola Modified Paths: -------------- trunk/pywikipedia/replace.py Modified: trunk/pywikipedia/replace.py =================================================================== --- trunk/pywikipedia/replace.py 2009-10-03 12:19:39 UTC (rev 7363) +++ trunk/pywikipedia/replace.py 2009-10-04 15:08:53 UTC (rev 7364) @@ -86,6 +86,8 @@ text. If the -regex argument is given, the first argument will be regarded as a regular expression, and the second argument might contain expressions like \\1 or \g<name>. + It is possible to introduce more than one pair of old text + and replacement. Examples: @@ -100,6 +102,9 @@ python replace.py -xml:foobar.xml "Errror" "Error" -namespace:0 +If you want to do more than one replacement at a time, use this: + python replace.py -xml:foobar.xml "Errror" "Error" "Faail" "Fail" -namespace:0 + If you have a page called 'John Doe' and want to fix the format of ISBNs, use: python replace.py -page:John_Doe -fix:isbn

14 years, 7 months

SVN: [7363] trunk/pywikipedia/welcome.py

by alexsh＠svn.wikimedia.org

Revision: 7363 Author: alexsh Date: 2009-10-03 12:19:39 +0000 (Sat, 03 Oct 2009) Log Message: ----------- variable typo Modified Paths: -------------- trunk/pywikipedia/welcome.py Modified: trunk/pywikipedia/welcome.py =================================================================== --- trunk/pywikipedia/welcome.py 2009-10-03 12:18:52 UTC (rev 7362) +++ trunk/pywikipedia/welcome.py 2009-10-03 12:19:39 UTC (rev 7363) @@ -612,7 +612,7 @@ return True def makelogpage(self, queue = []): - if not globalvar.makeWelcomLog: + if not globalvar.makeWelcomeLog: return None if len(queue) == 0: return None

14 years, 8 months

SVN: [7362] trunk/pywikipedia/welcome.py

by alexsh＠svn.wikimedia.org

Revision: 7362 Author: alexsh Date: 2009-10-03 12:18:52 +0000 (Sat, 03 Oct 2009) Log Message: ----------- use `not` to make more easy to turn down Modified Paths: -------------- trunk/pywikipedia/welcome.py Modified: trunk/pywikipedia/welcome.py =================================================================== --- trunk/pywikipedia/welcome.py 2009-10-03 09:02:59 UTC (rev 7361) +++ trunk/pywikipedia/welcome.py 2009-10-03 12:18:52 UTC (rev 7362) @@ -612,39 +612,40 @@ return True def makelogpage(self, queue = []): - if globalvar.makeWelcomLog: - if len(queue) == 0: - return None - - text = u'' - logg = wikipedia.translate(self.site, logbook) - target = logg + '/' + time.strftime('%Y/%m/%d', time.localtime(time.time())) - if self.site.lang == 'it': - target = logg + '/' + time.strftime('%d/%m/%Y', time.localtime(time.time())) - - logPage = wikipedia.Page(self.site, target) - if logPage.exists(): - text = logPage.get() - else: - #make new log page - showStatus() - wikipedia.output('Log page is not exist, getting information for page creation') - text = wikipedia.translate(self.site, logpage_header) - text += u'\n!%s' % self.site.namespace(2) - text += u'\n!%s' % string.capitalize(self.site.mediawiki_message('contribslink')) - - for result in queue: - # Adding the log... (don't take care of the variable's name...). - luser = wikipedia.url2link(result.name(), self.site, self.site) - text += u'\n{{WLE|user=%s|contribs=%d}}' % (luser, result.editCount()) - #update log page. - while True: - try: - logPage.put(text, wikipedia.translate(self.site, summary2) ) - return True - except wikipedia.EditConflict: - wikipedia.output(u'An edit conflict has occured. Pausing for 10 seconds before continuing.') - time.sleep(10) + if not globalvar.makeWelcomLog: + return None + if len(queue) == 0: + return None + + text = u'' + logg = wikipedia.translate(self.site, logbook) + target = logg + '/' + time.strftime('%Y/%m/%d', time.localtime(time.time())) + if self.site.lang == 'it': + target = logg + '/' + time.strftime('%d/%m/%Y', time.localtime(time.time())) + + logPage = wikipedia.Page(self.site, target) + if logPage.exists(): + text = logPage.get() + else: + #make new log page + showStatus() + wikipedia.output('Log page is not exist, getting information for page creation') + text = wikipedia.translate(self.site, logpage_header) + text += u'\n!%s' % self.site.namespace(2) + text += u'\n!%s' % string.capitalize(self.site.mediawiki_message('contribslink')) + + for result in queue: + # Adding the log... (don't take care of the variable's name...). + luser = wikipedia.url2link(result.name(), self.site, self.site) + text += u'\n{{WLE|user=%s|contribs=%d}}' % (luser, result.editCount()) + #update log page. + while True: + try: + logPage.put(text, wikipedia.translate(self.site, summary2) ) + return True + except wikipedia.EditConflict: + wikipedia.output(u'An edit conflict has occured. Pausing for 10 seconds before continuing.') + time.sleep(10) def parseNewUserLog(self): #if __name__ != '__main__':

14 years, 8 months

SVN: [7361] trunk/pywikipedia/welcome.py

by filnik＠svn.wikimedia.org

Revision: 7361 Author: filnik Date: 2009-10-03 09:02:59 +0000 (Sat, 03 Oct 2009) Log Message: ----------- Bugfix: the nlog functionality was broken Modified Paths: -------------- trunk/pywikipedia/welcome.py Modified: trunk/pywikipedia/welcome.py =================================================================== --- trunk/pywikipedia/welcome.py 2009-10-03 08:54:22 UTC (rev 7360) +++ trunk/pywikipedia/welcome.py 2009-10-03 09:02:59 UTC (rev 7361) @@ -612,38 +612,39 @@ return True def makelogpage(self, queue = []): - if len(queue) == 0: - return None - - text = u'' - logg = wikipedia.translate(self.site, logbook) - target = logg + '/' + time.strftime('%Y/%m/%d', time.localtime(time.time())) - if self.site.lang == 'it': - target = logg + '/' + time.strftime('%d/%m/%Y', time.localtime(time.time())) - - logPage = wikipedia.Page(self.site, target) - if logPage.exists(): - text = logPage.get() - else: - #make new log page - showStatus() - wikipedia.output('Log page is not exist, getting information for page creation') - text = wikipedia.translate(self.site, logpage_header) - text += u'\n!%s' % self.site.namespace(2) - text += u'\n!%s' % string.capitalize(self.site.mediawiki_message('contribslink')) - - for result in queue: - # Adding the log... (don't take care of the variable's name...). - luser = wikipedia.url2link(result.name(), self.site, self.site) - text += u'\n{{WLE|user=%s|contribs=%d}}' % (luser, result.editCount()) - #update log page. - while True: - try: - logPage.put(text, wikipedia.translate(self.site, summary2) ) - return True - except wikipedia.EditConflict: - wikipedia.output(u'An edit conflict has occured. Pausing for 10 seconds before continuing.') - time.sleep(10) + if globalvar.makeWelcomLog: + if len(queue) == 0: + return None + + text = u'' + logg = wikipedia.translate(self.site, logbook) + target = logg + '/' + time.strftime('%Y/%m/%d', time.localtime(time.time())) + if self.site.lang == 'it': + target = logg + '/' + time.strftime('%d/%m/%Y', time.localtime(time.time())) + + logPage = wikipedia.Page(self.site, target) + if logPage.exists(): + text = logPage.get() + else: + #make new log page + showStatus() + wikipedia.output('Log page is not exist, getting information for page creation') + text = wikipedia.translate(self.site, logpage_header) + text += u'\n!%s' % self.site.namespace(2) + text += u'\n!%s' % string.capitalize(self.site.mediawiki_message('contribslink')) + + for result in queue: + # Adding the log... (don't take care of the variable's name...). + luser = wikipedia.url2link(result.name(), self.site, self.site) + text += u'\n{{WLE|user=%s|contribs=%d}}' % (luser, result.editCount()) + #update log page. + while True: + try: + logPage.put(text, wikipedia.translate(self.site, summary2) ) + return True + except wikipedia.EditConflict: + wikipedia.output(u'An edit conflict has occured. Pausing for 10 seconds before continuing.') + time.sleep(10) def parseNewUserLog(self): #if __name__ != '__main__': @@ -984,4 +985,4 @@ f = file(filename, 'w') cPickle.dump(bot.welcomed_users, f) f.close() - wikipedia.stopme() \ No newline at end of file + wikipedia.stopme()

14 years, 8 months

SVN: [7360] trunk/pywikipedia

by alexsh＠svn.wikimedia.org

Revision: 7360 Author: alexsh Date: 2009-10-03 08:54:22 +0000 (Sat, 03 Oct 2009) Log Message: ----------- use large data post array to query redirects Modified Paths: -------------- trunk/pywikipedia/query.py trunk/pywikipedia/redirect.py Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2009-10-03 04:52:39 UTC (rev 7359) +++ trunk/pywikipedia/query.py 2009-10-03 08:54:22 UTC (rev 7360) @@ -59,18 +59,19 @@ # Titles param might be long, case convert it to post request data = None titlecount = 0 - if 'titles' in params: - titlecount = params['titles'].count('|') - if encodeTitle: - data = {'titles' : params['titles']} - del params['titles'] + for pLongKey in ['titles', 'pageids', 'ucusers']: # + if pLongKey in params: + titlecount = params[pLongKey].count('|') + if encodeTitle: + data = {pLongKey : params[pLongKey]} + del params[pLongKey] postAC = [ 'edit', 'login', 'purge', 'rollback', 'delete', 'undelete', 'protect', 'block', 'unblock', 'move', 'emailuser','import', 'userrights', ] if useAPI: - if params['action'] in postAC or data: + if params['action'] in postAC: path = site.api_address() else: path = site.api_address() + urllib.urlencode(params.items()) @@ -80,7 +81,7 @@ if wikipedia.verbose: if titlecount > 0: - wikipedia.output(u"Requesting %d titles from %s:%s" % (titlecount, site.lang, path)) + wikipedia.output(u"Requesting %d %s from %s:%s" % (titlecount, data.keys()[0], site.lang, path)) else: wikipedia.output(u"Request %s:%s" % (site.lang, path)) @@ -95,7 +96,7 @@ params["User-agent"] = useragent res = urllib2.urlopen(urllib2.Request(site.protocol() + '://' + site.hostname() + address, site.urlEncode(params))) jsontext = res.read() - elif params['action'] in postAC or data: + elif params['action'] in postAC: res, jsontext = site.postForm(path, params, sysop, site.cookies(sysop = sysop) ) else: if back_response: Modified: trunk/pywikipedia/redirect.py =================================================================== --- trunk/pywikipedia/redirect.py 2009-10-03 04:52:39 UTC (rev 7359) +++ trunk/pywikipedia/redirect.py 2009-10-03 08:54:22 UTC (rev 7360) @@ -310,15 +310,11 @@ namespaces = [ 0 ] #maxurllen = 1018 # accomodate "GET " + apiQ + CR + LF in 1024 bytes. apiQ = [] - maxurllen = 900 - len(self.site.hostname() + self.site.api_address() ) - clen = 0 for pageid in self.get_redirect_pageids_via_api(number, namespaces, start, until): apiQ.append(pageid) - clen += len(str(pageid)) + 1 - if clen > maxurllen and apiQ: + if len(apiQ) > 500: yield apiQ apiQ = [] - clen = 0 if apiQ: yield apiQ

14 years, 8 months

SVN: [7359] trunk/pywikipedia/redirect.py

by alexsh＠svn.wikimedia.org

Revision: 7359 Author: alexsh Date: 2009-10-03 04:52:39 +0000 (Sat, 03 Oct 2009) Log Message: ----------- RedirectGenerator().get_redirects_via_api(): change API Query type to JSON Modified Paths: -------------- trunk/pywikipedia/redirect.py Modified: trunk/pywikipedia/redirect.py =================================================================== --- trunk/pywikipedia/redirect.py 2009-10-02 20:34:20 UTC (rev 7358) +++ trunk/pywikipedia/redirect.py 2009-10-03 04:52:39 UTC (rev 7359) @@ -287,6 +287,8 @@ params['apfrom'] = start # print (apiQ) data = query.GetData(params, self.site) + if "limits" in data: # process aplimit = max + params['aplimit'] = int(data['limits']['allpages']) # wikipedia.output(u'===RESULT===\n%s\n' % result) for x in data['query']['allpages']: if until and x['title'] == until: @@ -299,30 +301,28 @@ break - def _next_redirects_via_api_commandline(self, apiQi, number = 'max', namespaces = [], - start = None, until = None ): + def _next_redirects_via_api_commandline(self, number = 'max', namespaces = [], start = None, until = None ): """ yields commands to the api for checking a set op page ids. """ # wikipedia.output(u'====> _next_redirects_via_api_commandline(apiQi=%s, number=%s, #ns=%d, start=%s, until=%s)' % (apiQi, number, len(namespaces), start, until)) if namespaces == []: namespaces = [ 0 ] - maxurllen = 1018 # accomodate "GET " + apiQ + CR + LF in 1024 bytes. - apiQ = '' + #maxurllen = 1018 # accomodate "GET " + apiQ + CR + LF in 1024 bytes. + apiQ = [] + maxurllen = 900 - len(self.site.hostname() + self.site.api_address() ) + clen = 0 for pageid in self.get_redirect_pageids_via_api(number, namespaces, start, until): - if apiQ: - tmp = ( '%s|%s' % ( apiQ, pageid ) ) - else: - tmp = ( '%s%s' % ( apiQi, pageid ) ) - if len(tmp) > maxurllen and apiQ: + apiQ.append(pageid) + clen += len(str(pageid)) + 1 + if clen > maxurllen and apiQ: yield apiQ - tmp = '' - apiQ = tmp + apiQ = [] + clen = 0 if apiQ: yield apiQ - def get_redirects_via_api(self, number = u'max', namespaces = [], start = None, - until = None, maxlen = 8 ): + def get_redirects_via_api(self, number = u'max', namespaces = [], start = None, until = None, maxlen = 8 ): """ Generator which will yield a tuple of data about Pages that are redirects: 0 - page title of a redirect page @@ -347,30 +347,26 @@ import urllib if namespaces == []: namespaces = [ 0 ] - apiQ1 = self.site.api_address() - apiQ1 += 'action=query' - apiQ1 += '&redirects' - apiQ1 += '&format=xml' - apiQ1 += '&pageids=' - redirectRe = re.compile('<r from="(.*?)" to="(.*?)"') - missingpageRe = re.compile('<page .*? title="(.*?)" missing=""') - existingpageRe = re.compile('<page pageid=".*?" .*? title="(.*?)"') - for apiQ in self._next_redirects_via_api_commandline(apiQ1, number = number, - namespaces = namespaces, start = start, until = until ): + params = { + 'action':'query', + 'redirects':1, + #'':'', + } + for apiQ in self._next_redirects_via_api_commandline(number, namespaces, start, until): # wikipedia.output (u'===apiQ=%s' % apiQ) - result = self.site.getUrl(apiQ) + params['pageids'] = query.ListToParam(apiQ) + data = query.GetData(params, self.site) # wikipedia.output(u'===RESULT===\n%s\n' % result) redirects = {} pages = {} - for redirect in redirectRe.findall(result): - # wikipedia.output (u'R: %s => %s' % redirect) - redirects[redirect[0]] = redirect[1] - for pagetitle in missingpageRe.findall(result): + redirects = dict([[x['from'], x['to']] for x in data['query']['redirects']]) + + for pagetitle in data['query']['pages'].values(): # wikipedia.output (u'M: %s' % pagetitle) - pages[pagetitle] = False - for pagetitle in existingpageRe.findall(result): - # wikipedia.output (u'P: %s' % pagetitle) - pages[pagetitle] = True + if 'missing' in pagetitle and 'pageid' not in pagetitle: + pages[pagetitle['title']] = False + else: + pages[pagetitle['title']] = True for redirect in redirects: target = redirects[redirect] result = 0

14 years, 8 months

SVN: [7358] trunk/pywikipedia/redirect.py

by alexsh＠svn.wikimedia.org

Revision: 7358 Author: alexsh Date: 2009-10-02 20:34:20 +0000 (Fri, 02 Oct 2009) Log Message: ----------- RedirectGenerator().get_redirect_pageids_via_api(): change API Query type to JSON(tested ok) Modified Paths: -------------- trunk/pywikipedia/redirect.py Modified: trunk/pywikipedia/redirect.py =================================================================== --- trunk/pywikipedia/redirect.py 2009-10-02 20:12:56 UTC (rev 7357) +++ trunk/pywikipedia/redirect.py 2009-10-02 20:34:20 UTC (rev 7358) @@ -61,7 +61,7 @@ # # from __future__ import generators -import wikipedia, config +import wikipedia, config, query import xmlreader import re, sys @@ -259,8 +259,7 @@ else: return redict - def get_redirect_pageids_via_api(self, number = u'max', namespaces = [], - start = None, until = None ): + def get_redirect_pageids_via_api(self, number = u'max', namespaces = [], start = None, until = None ): """ Generator which will yield page IDs of Pages that are redirects. Get number of page ids in one go. @@ -268,45 +267,37 @@ In each namespace, start alphabetically from a pagetitle start, wich need not exist. """ # wikipedia.output(u'====> get_redirect_pageids_via_api(number=%s, #ns=%d, start=%s, until=%s)' % (number, len(namespaces), start, until)) - import urllib if namespaces == []: namespaces = [ 0 ] - apiQ0 = self.site.api_address() - apiQ0 += 'action=query' - apiQ0 += '&list=allpages' - apiQ0 += '&apfilterredir=redirects' - apiQ0 += '&aplimit=%s' % number - apiQ0 += '&format=xml' - apPageTitleRe = re.compile(' pageid="(.*?)" .*? title="(.*?)"') - apPageIdRe = re.compile(' pageid="(.*?)"') - apfromRe = re.compile(' apfrom="(.*?)"') + params = { + 'action':'query', + 'list':'allpages', + 'apfilterredir':'redirects', + 'aplimit':number, + 'apdir':'ascending', + #'':'', + } + for ns in namespaces: # print (ns) - apiQns = apiQ0 + '&apnamespace=%s' % ns + params['apnamespace'] = ns # print (apiQns) - while apiQns: - apiQ = apiQns + while True: if start: - apiQ += '&apfrom=%s' % urllib.quote(start.encode(site.encoding())) + params['apfrom'] = start # print (apiQ) - result = site.getUrl(apiQ) + data = query.GetData(params, self.site) # wikipedia.output(u'===RESULT===\n%s\n' % result) - if until: - for (pageid, pagetitle) in apPageTitleRe.findall(result): - # wikipedia.output(u'===PAGEID=%s: %s' % (pageid, pagetitle)) ## TODO: make this a -verbose mode output, independant of -until - if pagetitle > until: - apiQns = None - break - yield pageid + for x in data['query']['allpages']: + if until and x['title'] == until: + break + yield x['pageid'] + + if 'query-continue' in data: + params['apfrom'] = data['query-continue']['allpages']['apfrom'] else: - for pageid in apPageIdRe.findall(result): - # wikipedia.output(u'===PAGEID=%s' % pageid) - yield pageid - m = apfromRe.search(result) - if m: - start = m.group(1) - else: break + def _next_redirects_via_api_commandline(self, apiQi, number = 'max', namespaces = [], start = None, until = None ): @@ -318,8 +309,7 @@ namespaces = [ 0 ] maxurllen = 1018 # accomodate "GET " + apiQ + CR + LF in 1024 bytes. apiQ = '' - for pageid in self.get_redirect_pageids_via_api(number = number, namespaces = namespaces, - start = start, until = until ): + for pageid in self.get_redirect_pageids_via_api(number, namespaces, start, until): if apiQ: tmp = ( '%s|%s' % ( apiQ, pageid ) ) else:

14 years, 8 months

SVN: [7357] trunk/pywikipedia/redirect.py

by alexsh＠svn.wikimedia.org

Revision: 7357 Author: alexsh Date: 2009-10-02 20:12:56 +0000 (Fri, 02 Oct 2009) Log Message: ----------- redirect optimize: add self.site to replace wikipedia.getSite() in functions Modified Paths: -------------- trunk/pywikipedia/redirect.py Modified: trunk/pywikipedia/redirect.py =================================================================== --- trunk/pywikipedia/redirect.py 2009-10-02 11:53:53 UTC (rev 7356) +++ trunk/pywikipedia/redirect.py 2009-10-02 20:12:56 UTC (rev 7357) @@ -179,6 +179,8 @@ def __init__(self, xmlFilename=None, namespaces=[], offset=-1, use_move_log=False, use_api=False, start=None, until=None, number=None): + self.site = wikipedia.getSite() + self.xmlFilename = xmlFilename self.namespaces = namespaces self.offset = offset @@ -199,8 +201,7 @@ redict = {} # open xml dump and read page titles out of it dump = xmlreader.XmlDump(xmlFilename) - site = wikipedia.getSite() - redirR = site.redirectRegex() + redirR = self.site.redirectRegex() readPagesCount = 0 if alsoGetPageTitles: pageTitles = set() @@ -210,7 +211,7 @@ if readPagesCount % 10000 == 0: wikipedia.output(u'%i pages read...' % readPagesCount) if len(self.namespaces) > 0: - if wikipedia.Page(site, entry.title).namespace() \ + if wikipedia.Page(self.site, entry.title).namespace() \ not in self.namespaces: continue if alsoGetPageTitles: @@ -220,10 +221,10 @@ if m: target = m.group(1) # There might be redirects to another wiki. Ignore these. - for code in site.family.langs.keys(): + for code in self.site.family.langs.keys(): if target.startswith('%s:' % code) \ or target.startswith(':%s:' % code): - if code == site.language(): + if code == self.site.language(): # link to our wiki, but with the lang prefix target = target[(len(code)+1):] if target.startswith(':'): @@ -258,7 +259,7 @@ else: return redict - def get_redirect_pageids_via_api(self, number = u'max', namespaces = [], site = None, + def get_redirect_pageids_via_api(self, number = u'max', namespaces = [], start = None, until = None ): """ Generator which will yield page IDs of Pages that are redirects. @@ -268,11 +269,9 @@ """ # wikipedia.output(u'====> get_redirect_pageids_via_api(number=%s, #ns=%d, start=%s, until=%s)' % (number, len(namespaces), start, until)) import urllib - if site is None: - site = wikipedia.getSite() if namespaces == []: namespaces = [ 0 ] - apiQ0 = site.api_address() + apiQ0 = self.site.api_address() apiQ0 += 'action=query' apiQ0 += '&list=allpages' apiQ0 += '&apfilterredir=redirects' @@ -310,19 +309,17 @@ break def _next_redirects_via_api_commandline(self, apiQi, number = 'max', namespaces = [], - site = None, start = None, until = None ): + start = None, until = None ): """ yields commands to the api for checking a set op page ids. """ # wikipedia.output(u'====> _next_redirects_via_api_commandline(apiQi=%s, number=%s, #ns=%d, start=%s, until=%s)' % (apiQi, number, len(namespaces), start, until)) - if site is None: - site = wikipedia.getSite() if namespaces == []: namespaces = [ 0 ] maxurllen = 1018 # accomodate "GET " + apiQ + CR + LF in 1024 bytes. apiQ = '' for pageid in self.get_redirect_pageids_via_api(number = number, namespaces = namespaces, - site = site, start = start, until = until ): + start = start, until = until ): if apiQ: tmp = ( '%s|%s' % ( apiQ, pageid ) ) else: @@ -334,7 +331,7 @@ if apiQ: yield apiQ - def get_redirects_via_api(self, number = u'max', namespaces = [], site = None, start = None, + def get_redirects_via_api(self, number = u'max', namespaces = [], start = None, until = None, maxlen = 8 ): """ Generator which will yield a tuple of data about Pages that are redirects: @@ -358,11 +355,9 @@ """ # wikipedia.output(u'====> get_redirects_via_api(number=%s, #ns=%d, start=%s, until=%s, maxlen=%s)' % (number, len(namespaces), start, until, maxlen)) import urllib - if site is None: - site = wikipedia.getSite() if namespaces == []: namespaces = [ 0 ] - apiQ1 = site.api_address() + apiQ1 = self.site.api_address() apiQ1 += 'action=query' apiQ1 += '&redirects' apiQ1 += '&format=xml' @@ -371,9 +366,9 @@ missingpageRe = re.compile('<page .*? title="(.*?)" missing=""') existingpageRe = re.compile('<page pageid=".*?" .*? title="(.*?)"') for apiQ in self._next_redirects_via_api_commandline(apiQ1, number = number, - namespaces = namespaces, site = site, start = start, until = until ): + namespaces = namespaces, start = start, until = until ): # wikipedia.output (u'===apiQ=%s' % apiQ) - result = site.getUrl(apiQ) + result = self.site.getUrl(apiQ) # wikipedia.output(u'===RESULT===\n%s\n' % result) redirects = {} pages = {} @@ -408,11 +403,10 @@ def retrieve_broken_redirects(self): if self.use_api: - mysite = wikipedia.getSite() count = 0 for (pagetitle, type, target, final) in self.get_redirects_via_api( namespaces = self.namespaces, - site = mysite, start = self.api_start, + start = self.api_start, until = self.api_until, maxlen = 2): if type == 0: yield pagetitle @@ -423,11 +417,10 @@ elif self.xmlFilename == None: # retrieve information from the live wiki's maintenance page - mysite = wikipedia.getSite() # broken redirect maintenance page's URL - path = mysite.broken_redirects_address(default_limit = False) + path = self.site.broken_redirects_address(default_limit = False) wikipedia.output(u'Retrieving special page...') - maintenance_txt = mysite.getUrl(path) + maintenance_txt = self.site.getUrl(path) # regular expression which finds redirects which point to a # non-existing page inside the HTML @@ -450,11 +443,10 @@ def retrieve_double_redirects(self): if self.use_api: - mysite = wikipedia.getSite() count = 0 for (pagetitle, type, target, final) in self.get_redirects_via_api( namespaces = self.namespaces, - site = mysite, start = self.api_start, + start = self.api_start, until = self.api_until, maxlen = 2): if type != 0 and type != 1: yield pagetitle @@ -468,13 +460,12 @@ for redir_page in self.get_moved_pages_redirects(): yield redir_page.title() return - mysite = wikipedia.getSite() # retrieve information from the live wiki's maintenance page # double redirect maintenance page's URL # wikipedia.config.special_page_limit = 1000 - path = mysite.double_redirects_address(default_limit = False) + path = self.site.double_redirects_address(default_limit = False) wikipedia.output(u'Retrieving special page...') - maintenance_txt = mysite.getUrl(path) + maintenance_txt = self.site.getUrl(path) # regular expression which finds redirects which point to # another redirect inside the HTML @@ -500,9 +491,7 @@ wiki = re.escape(wikipedia.getSite().nice_get_address('')) # /w/index.php index = re.escape(wikipedia.getSite().path()) - move_regex = re.compile( - r'moved <a href.*?>(.*?)</a> to <a href=.*?>.*?</a>.*?</li>' - ) + move_regex = re.compile(r'moved <a href.*?>(.*?)</a> to <a href=.*?>.*?</a>.*?</li>') def get_moved_pages_redirects(self): '''generate redirects to recently-moved pages''' @@ -511,19 +500,17 @@ if self.offset <= 0: self.offset = 1 - offsetpattern = re.compile( -r"""\(<a href="/w/index\.php\?title=Special:Log&offset=(\d+)&limit=500&type=move" title="Special:Log" rel="next">older 500</a>\)""") + offsetpattern = re.compile(r"""\(<a href="/w/index\.php\?title=Special:Log&offset=(\d+)&limit=500&type=move" title="Special:Log" rel="next">older 500</a>\)""") start = datetime.datetime.utcnow() \ - datetime.timedelta(0, self.offset*3600) # self.offset hours ago offset_time = start.strftime("%Y%m%d%H%M%S") - site = wikipedia.getSite() while True: move_url = \ - site.path() + "?title=Special:Log&limit=500&offset=%s&type=move"\ + self.site.path() + "?title=Special:Log&limit=500&offset=%s&type=move"\ % offset_time try: - move_list = site.getUrl(move_url) + move_list = self.site.getUrl(move_url) if wikipedia.verbose: wikipedia.output(u"[%s]" % offset_time) except: @@ -534,7 +521,7 @@ if wikipedia.verbose: wikipedia.output(u"%s moved pages" % len(g)) for moved_title in g: - moved_page = wikipedia.Page(site, moved_title) + moved_page = wikipedia.Page(self.site, moved_title) try: if not moved_page.isRedirectPage(): continue @@ -545,8 +532,7 @@ # moved_page is now a redirect, so any redirects pointing # to it need to be changed try: - for page in moved_page.getReferences(follow_redirects=True, - redirectsOnly=True): + for page in moved_page.getReferences(follow_redirects=True, redirectsOnly=True): yield page except wikipedia.NoPage: # original title must have been deleted after move @@ -558,6 +544,9 @@ class RedirectRobot: def __init__(self, action, generator, always=False, number=None): + + self.site = wikipedia.getSite() + self.action = action self.generator = generator self.always = always @@ -578,20 +567,18 @@ return True def delete_broken_redirects(self): - mysite = wikipedia.getSite() # get reason for deletion text - reason = wikipedia.translate(mysite, reason_broken) + reason = wikipedia.translate(self.site, reason_broken) for redir_name in self.generator.retrieve_broken_redirects(): - self.delete_1_broken_redirect(mysite, redir_name, reason) + self.delete_1_broken_redirect( redir_name, reason) if self.exiting: break - def delete_1_broken_redirect(self, mysite, redir_name, reason): - redir_page = wikipedia.Page(mysite, redir_name) + def delete_1_broken_redirect(self, redir_name, reason): + redir_page = wikipedia.Page(self.site, redir_name) # Show the title of the page we're working on. # Highlight the title in purple. - wikipedia.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" - % redir_page.title()) + wikipedia.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" % redir_page.title()) try: targetPage = redir_page.getRedirectTarget() except wikipedia.IsNotRedirectPage: @@ -625,19 +612,17 @@ wikipedia.output(u'') def fix_double_redirects(self): - mysite = wikipedia.getSite() - summary = wikipedia.translate(mysite, msg_double) + summary = wikipedia.translate(self.site, msg_double) for redir_name in self.generator.retrieve_double_redirects(): - self.fix_1_double_redirect(mysite, redir_name, summary) + self.fix_1_double_redirect(redir_name, summary) if self.exiting: break - def fix_1_double_redirect(self, mysite, redir_name, summary): - redir = wikipedia.Page(mysite, redir_name) + def fix_1_double_redirect(self, redir_name, summary): + redir = wikipedia.Page(self.site, redir_name) # Show the title of the page we're working on. # Highlight the title in purple. - wikipedia.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" - % redir.title()) + wikipedia.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" % redir.title()) newRedir = redir redirList = [] # bookkeeping to detect loops while True: @@ -684,9 +669,8 @@ wikipedia.output( u' Links to: %s.' % targetPage.aslink()) - if targetPage.site() != mysite: - wikipedia.output( - u'Warning: redirect target (%s) is on a different site.' + if targetPage.site() != self.site: + wikipedia.output(u'Warning: redirect target (%s) is on a different site.' % (targetPage.aslink())) if self.always: break # skip if automatic @@ -710,10 +694,8 @@ and targetPage.site().lang in sd_tagging_sum: wikipedia.output(u"Tagging redirect for deletion") # Delete the two redirects - content = wikipedia.translate(targetPage.site().lang, - sd_template)+"\n"+content - summ = wikipedia.translate(targetPage.site().lang, - sd_tagging_sum) + content = wikipedia.translate(targetPage.site().lang, sd_template)+"\n"+content + summ = wikipedia.translate(targetPage.site().lang, sd_tagging_sum) targetPage.put(content, summ) redir.put(content, summ) else: @@ -726,9 +708,9 @@ except wikipedia.BadTitle: wikipedia.output(u"Bad Title Error") break - text = mysite.redirectRegex().sub( + text = self.site.redirectRegex().sub( '#%s %s' % - (mysite.redirect( True ), + (self.site.redirect( True ), targetPage.aslink()), oldText) if text == oldText: @@ -740,40 +722,36 @@ except wikipedia.LockedPage: wikipedia.output(u'%s is locked.' % redir.title()) except wikipedia.SpamfilterError, error: - wikipedia.output( -u"Saving page [[%s]] prevented by spam filter: %s" + wikipedia.output(u"Saving page [[%s]] prevented by spam filter: %s" % (redir.title(), error.url)) except wikipedia.PageNotSaved, error: wikipedia.output(u"Saving page [[%s]] failed: %s" % (redir.title(), error)) except wikipedia.NoUsername: - wikipedia.output( -u"Page [[%s]] not saved; sysop privileges required." + wikipedia.output(u"Page [[%s]] not saved; sysop privileges required." % redir.title()) except wikipedia.Error, error: - wikipedia.output( -u"Unexpected error occurred trying to save [[%s]]: %s" + wikipedia.output(u"Unexpected error occurred trying to save [[%s]]: %s" % (redir.title(), error)) break def fix_double_or_delete_broken_redirects(self): # TODO: part of this should be moved to generator, the rest merged into self.run() - mysite = wikipedia.getSite() # get reason for deletion text - delete_reason = wikipedia.translate(mysite, reason_broken) - double_summary = wikipedia.translate(mysite, msg_double) + delete_reason = wikipedia.translate(self.site, reason_broken) + double_summary = wikipedia.translate(self.site, msg_double) count = 0 for (redir_name, code, target, final) in self.generator.get_redirects_via_api( namespaces = self.generator.namespaces, - site = mysite, start = self.generator.api_start, + start = self.generator.api_start, until = self.generator.api_until, maxlen = 2): if code == 1: continue elif code == 0: - self.delete_1_broken_redirect(mysite, redir_name, delete_reason) + self.delete_1_broken_redirect(redir_name, delete_reason) count += 1 else: - self.fix_1_double_redirect(mysite, redir_name, double_summary) + self.fix_1_double_redirect(redir_name, double_summary) count += 1 # print ('%s .. %s' % (count, self.number)) if self.exiting or ( self.number and count >= self.number ): @@ -785,7 +763,7 @@ if self.action == 'double': # get summary text wikipedia.setAction( - wikipedia.translate(wikipedia.getSite(), msg_double)) + wikipedia.translate(self.site, msg_double)) self.fix_double_redirects() elif self.action == 'broken': self.delete_broken_redirects()

14 years, 8 months

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn October 2009