Revision: 7131 Author: alexsh Date: 2009-08-08 01:00:08 +0000 (Sat, 08 Aug 2009)
Log Message: ----------- Change all simplejson.load() datas to use query.getData()
Modified Paths: -------------- trunk/pywikipedia/archivebot.py trunk/pywikipedia/category_redirect.py trunk/pywikipedia/commonsdelinker/image_replacer.py trunk/pywikipedia/copyright_clean.py trunk/pywikipedia/revertbot.py
Modified: trunk/pywikipedia/archivebot.py =================================================================== --- trunk/pywikipedia/archivebot.py 2009-08-08 00:42:34 UTC (rev 7130) +++ trunk/pywikipedia/archivebot.py 2009-08-08 01:00:08 UTC (rev 7131) @@ -34,10 +34,10 @@ # __version__ = '$Id$' # -import wikipedia, pagegenerators +import wikipedia, pagegenerators, query Site = wikipedia.getSite()
-import os, re, time, locale, traceback, string, urllib, simplejson +import os, re, time, locale, traceback, string, urllib
try: #Get a constructor for the MD5 hash object import hashlib @@ -217,8 +217,7 @@ qdata['eicontinue'] = eicontinue
wikipedia.output(u'Fetching template transclusions...') - response, data = Site.postData(Site.apipath(), urllib.urlencode(qdata)) - result = simplejson.loads(data) + response, result = query.GetData(qdata, Site, back_response = True)
for page_d in result['query']['embeddedin']: yield wikipedia.Page(Site, page_d['title'])
Modified: trunk/pywikipedia/category_redirect.py =================================================================== --- trunk/pywikipedia/category_redirect.py 2009-08-08 00:42:34 UTC (rev 7130) +++ trunk/pywikipedia/category_redirect.py 2009-08-08 01:00:08 UTC (rev 7131) @@ -14,9 +14,7 @@ """ __version__ = '$Id$'
-import wikipedia, catlib -import pagegenerators -import simplejson +import wikipedia, catlib, query, pagegenerators import cPickle import math import re @@ -302,11 +300,9 @@
def query_results(self, **data): """Iterate results from API action=query, using data as parameters.""" - addr = self.site.apipath() querydata = {'action': 'query', - 'format': 'json', 'maxlag': str(wikipedia.config.maxlag)} - querydata.update(data) + querydata = query.CombineParams(querydata, data) if not querydata.has_key("action")\ or not querydata['action'] == 'query': raise ValueError( @@ -314,19 +310,18 @@ ) waited = 0 while True: - response, data = self.site.postForm(addr, querydata) - if response.status != 200: - # WARNING: if the server is down, this could - # cause an infinite loop - wikipedia.output(u"HTTP error %i received; retrying..." - % response.status) - time.sleep(5) - continue - if data.startswith(u"unknown_action"): - e = {'code': data[:14], 'info': data[16:]} - raise APIError(e) try: - result = simplejson.loads(data) + response, result = query.GetData(predata, self,site back_response = True) + if response.status != 200: + # WARNING: if the server is down, this could + # cause an infinite loop + wikipedia.output(u"HTTP error %i received; retrying..." + % response.status) + time.sleep(5) + continue + if data.startswith(u"unknown_action"): + e = {'code': data[:14], 'info': data[16:]} + raise APIError(e) except ValueError: # if the result isn't valid JSON, there must be a server # problem. Wait a few seconds and try again
Modified: trunk/pywikipedia/commonsdelinker/image_replacer.py =================================================================== --- trunk/pywikipedia/commonsdelinker/image_replacer.py 2009-08-08 00:42:34 UTC (rev 7130) +++ trunk/pywikipedia/commonsdelinker/image_replacer.py 2009-08-08 01:00:08 UTC (rev 7131) @@ -10,7 +10,7 @@ # Distributed under the terms of the MIT license. # __version__ = '$Id$' -import config, wikipedia, simplejson +import config, wikipedia, query import re, time import threadpool import sys, os, signal, traceback @@ -149,21 +149,19 @@ """ Fetch the last 50 revisions using the API """
address = self.site.api_address() - predata = [ - ('action', 'query'), - ('prop', 'revisions'), - ('titles', title.encode('utf-8')), - ('rvprop', 'timestamp|user|comment|content'), - ('rvlimit', '50'), - ('format', 'json'), - ] + predata = { + 'action': 'query', + 'prop': 'revisions', + 'titles': title.encode('utf-8'), + 'rvprop': 'timestamp|user|comment|content', + 'rvlimit': '50', + } if username: - predata.append(('rvexcludeuser', username.encode('utf-8'))) + predata['rvexcludeuser'] = username.encode('utf-8') if since: - predata.append(('rvend', since)) - response, data = self.site.postForm(address, predata) - data = simplejson.loads(data) - if 'error' in data: + predata['rvend'] = since + response, data = query.GetData(predata, self.site, back_response = True) + if data.has_key('error'): raise RuntimeError(data['error'])
page = data['query']['pages'].values()[0]
Modified: trunk/pywikipedia/copyright_clean.py =================================================================== --- trunk/pywikipedia/copyright_clean.py 2009-08-08 00:42:34 UTC (rev 7130) +++ trunk/pywikipedia/copyright_clean.py 2009-08-08 01:00:08 UTC (rev 7131) @@ -9,7 +9,7 @@ # Distributed under the terms of the MIT license. #
-import httplib, socket, simplejson, re, time +import httplib, socket, re, time import config, wikipedia, catlib, pagegenerators, query
from urllib import urlencode @@ -33,40 +33,22 @@ rev_templateC = re.compile("(?m)^(?:{{/t|.*?}}\n?)?{{(?:/box|botbox)|.*?|(.*?)|")
def query_api(data): - predata = [ - ('format', 'json'), - ('action', 'query'), - ('prop', 'revisions'), - data] - data = urlencode(predata) - host = wikipedia.getSite().hostname() - address = wikipedia.getSite().api_address() - conn = httplib.HTTPConnection(host) - conn.request("GET", address + data) - response = conn.getresponse() - data = response.read() - conn.close() - return data + predata = { + 'action': 'query', + 'prop': 'revisions', + } + predata = query.CombineParams(predata, data) + return query.GetData(predata)
def query_old_api(data):
- predata = [ - ('format', 'json'), - ('what', 'revisions'), - ('rvlimit', '1'), - data] + predata = { + 'what': 'revisions', + 'rvlimit': '1', + } + predata = query.CombineParams(predata, data) + return query.GetData(predata, useAPI = False)
- data = urlencode(predata) - host = wikipedia.getSite().hostname() - address = wikipedia.getSite().query_address() - conn = httplib.HTTPConnection(host) - conn.request("GET", address + data) - response = conn.getresponse() - data = response.read() - conn.close() - - return data - def old_page_exist(title): for pageobjs in query_results_titles: for key in pageobjs['pages']: @@ -135,9 +117,9 @@
# No more of 50 titles at a time using API for s in mysplit(query.ListToParam(titles), 50, "|"): - query_results_titles.append(simplejson.loads(query_api(('titles', s)))) + query_results_titles.append(query_api({'titles': s,})) for s in mysplit(query.ListToParam(revids), 50, "|"): - query_results_revids.append(simplejson.loads(query_api(('revids', s)))) + query_results_revids.append(query_api({'revids': ,}))
comment_entry = list() add_separator = False
Modified: trunk/pywikipedia/revertbot.py =================================================================== --- trunk/pywikipedia/revertbot.py 2009-08-08 00:42:34 UTC (rev 7130) +++ trunk/pywikipedia/revertbot.py 2009-08-08 01:00:08 UTC (rev 7131) @@ -1,5 +1,4 @@ -import wikipedia -import simplejson +import wikipedia, query
__version__ = '$Id$'
@@ -18,7 +17,6 @@ self.comment = comment
def get_contributions(self, max = -1, ns = None): - address = self.site.api_address() predata = { 'action': 'query', 'list': 'usercontribs', @@ -37,12 +35,11 @@ item = iterator.next() except StopIteration: self.log(u'Fetching new batch of contributions') - response, data = self.site.postForm(address, predata) - data = simplejson.loads(data) - if 'error' in data: + response, data = query.GetData(predata, self.site, back_response = True) + if data.has_key('error'): raise RuntimeError(data['error']) - if 'usercontribs' in data.get('query-continue', ()): - predata.update(data['query-continue']['usercontribs']) + if data.has_key('query-continue'): + predata['uccontinue'] = data['query-continue']['usercontribs'] else: never_continue = True iterator = iter(data['query']['usercontribs']) @@ -81,10 +78,9 @@ 'rvstart': item['timestamp'], 'format': 'json' } - response, data = self.site.postForm(self.site.api_address(), predata) - data = simplejson.loads(data) + response, data = query.GetData(predata, self.site, back_response = True)
- if 'error' in data: + if data.has_key('error'): raise RuntimeError(data['error'])
pages = data['query'].get('pages', ())