Revision: 7360 Author: alexsh Date: 2009-10-03 08:54:22 +0000 (Sat, 03 Oct 2009)
Log Message: ----------- use large data post array to query redirects
Modified Paths: -------------- trunk/pywikipedia/query.py trunk/pywikipedia/redirect.py
Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2009-10-03 04:52:39 UTC (rev 7359) +++ trunk/pywikipedia/query.py 2009-10-03 08:54:22 UTC (rev 7360) @@ -59,18 +59,19 @@ # Titles param might be long, case convert it to post request data = None titlecount = 0 - if 'titles' in params: - titlecount = params['titles'].count('|') - if encodeTitle: - data = {'titles' : params['titles']} - del params['titles'] + for pLongKey in ['titles', 'pageids', 'ucusers']: # + if pLongKey in params: + titlecount = params[pLongKey].count('|') + if encodeTitle: + data = {pLongKey : params[pLongKey]} + del params[pLongKey]
postAC = [ 'edit', 'login', 'purge', 'rollback', 'delete', 'undelete', 'protect', 'block', 'unblock', 'move', 'emailuser','import', 'userrights', ] if useAPI: - if params['action'] in postAC or data: + if params['action'] in postAC: path = site.api_address() else: path = site.api_address() + urllib.urlencode(params.items()) @@ -80,7 +81,7 @@
if wikipedia.verbose: if titlecount > 0: - wikipedia.output(u"Requesting %d titles from %s:%s" % (titlecount, site.lang, path)) + wikipedia.output(u"Requesting %d %s from %s:%s" % (titlecount, data.keys()[0], site.lang, path)) else: wikipedia.output(u"Request %s:%s" % (site.lang, path))
@@ -95,7 +96,7 @@ params["User-agent"] = useragent res = urllib2.urlopen(urllib2.Request(site.protocol() + '://' + site.hostname() + address, site.urlEncode(params))) jsontext = res.read() - elif params['action'] in postAC or data: + elif params['action'] in postAC: res, jsontext = site.postForm(path, params, sysop, site.cookies(sysop = sysop) ) else: if back_response:
Modified: trunk/pywikipedia/redirect.py =================================================================== --- trunk/pywikipedia/redirect.py 2009-10-03 04:52:39 UTC (rev 7359) +++ trunk/pywikipedia/redirect.py 2009-10-03 08:54:22 UTC (rev 7360) @@ -310,15 +310,11 @@ namespaces = [ 0 ] #maxurllen = 1018 # accomodate "GET " + apiQ + CR + LF in 1024 bytes. apiQ = [] - maxurllen = 900 - len(self.site.hostname() + self.site.api_address() ) - clen = 0 for pageid in self.get_redirect_pageids_via_api(number, namespaces, start, until): apiQ.append(pageid) - clen += len(str(pageid)) + 1 - if clen > maxurllen and apiQ: + if len(apiQ) > 500: yield apiQ apiQ = [] - clen = 0 if apiQ: yield apiQ