Revision: 7131
Author: alexsh
Date: 2009-08-08 01:00:08 +0000 (Sat, 08 Aug 2009)
Log Message:
-----------
Change all simplejson.load() datas to use query.getData()
Modified Paths:
--------------
trunk/pywikipedia/archivebot.py
trunk/pywikipedia/category_redirect.py
trunk/pywikipedia/commonsdelinker/image_replacer.py
trunk/pywikipedia/copyright_clean.py
trunk/pywikipedia/revertbot.py
Modified: trunk/pywikipedia/archivebot.py
===================================================================
--- trunk/pywikipedia/archivebot.py 2009-08-08 00:42:34 UTC (rev 7130)
+++ trunk/pywikipedia/archivebot.py 2009-08-08 01:00:08 UTC (rev 7131)
@@ -34,10 +34,10 @@
#
__version__ = '$Id$'
#
-import wikipedia, pagegenerators
+import wikipedia, pagegenerators, query
Site = wikipedia.getSite()
-import os, re, time, locale, traceback, string, urllib, simplejson
+import os, re, time, locale, traceback, string, urllib
try: #Get a constructor for the MD5 hash object
import hashlib
@@ -217,8 +217,7 @@
qdata['eicontinue'] = eicontinue
wikipedia.output(u'Fetching template transclusions...')
- response, data = Site.postData(Site.apipath(), urllib.urlencode(qdata))
- result = simplejson.loads(data)
+ response, result = query.GetData(qdata, Site, back_response = True)
for page_d in result['query']['embeddedin']:
yield wikipedia.Page(Site, page_d['title'])
Modified: trunk/pywikipedia/category_redirect.py
===================================================================
--- trunk/pywikipedia/category_redirect.py 2009-08-08 00:42:34 UTC (rev 7130)
+++ trunk/pywikipedia/category_redirect.py 2009-08-08 01:00:08 UTC (rev 7131)
@@ -14,9 +14,7 @@
"""
__version__ = '$Id$'
-import wikipedia, catlib
-import pagegenerators
-import simplejson
+import wikipedia, catlib, query, pagegenerators
import cPickle
import math
import re
@@ -302,11 +300,9 @@
def query_results(self, **data):
"""Iterate results from API action=query, using data as parameters."""
- addr = self.site.apipath()
querydata = {'action': 'query',
- 'format': 'json',
'maxlag': str(wikipedia.config.maxlag)}
- querydata.update(data)
+ querydata = query.CombineParams(querydata, data)
if not querydata.has_key("action")\
or not querydata['action'] == 'query':
raise ValueError(
@@ -314,19 +310,18 @@
)
waited = 0
while True:
- response, data = self.site.postForm(addr, querydata)
- if response.status != 200:
- # WARNING: if the server is down, this could
- # cause an infinite loop
- wikipedia.output(u"HTTP error %i received; retrying..."
- % response.status)
- time.sleep(5)
- continue
- if data.startswith(u"unknown_action"):
- e = {'code': data[:14], 'info': data[16:]}
- raise APIError(e)
try:
- result = simplejson.loads(data)
+ response, result = query.GetData(predata, self,site back_response = True)
+ if response.status != 200:
+ # WARNING: if the server is down, this could
+ # cause an infinite loop
+ wikipedia.output(u"HTTP error %i received; retrying..."
+ % response.status)
+ time.sleep(5)
+ continue
+ if data.startswith(u"unknown_action"):
+ e = {'code': data[:14], 'info': data[16:]}
+ raise APIError(e)
except ValueError:
# if the result isn't valid JSON, there must be a server
# problem. Wait a few seconds and try again
Modified: trunk/pywikipedia/commonsdelinker/image_replacer.py
===================================================================
--- trunk/pywikipedia/commonsdelinker/image_replacer.py 2009-08-08 00:42:34 UTC (rev 7130)
+++ trunk/pywikipedia/commonsdelinker/image_replacer.py 2009-08-08 01:00:08 UTC (rev 7131)
@@ -10,7 +10,7 @@
# Distributed under the terms of the MIT license.
#
__version__ = '$Id$'
-import config, wikipedia, simplejson
+import config, wikipedia, query
import re, time
import threadpool
import sys, os, signal, traceback
@@ -149,21 +149,19 @@
""" Fetch the last 50 revisions using the API """
address = self.site.api_address()
- predata = [
- ('action', 'query'),
- ('prop', 'revisions'),
- ('titles', title.encode('utf-8')),
- ('rvprop', 'timestamp|user|comment|content'),
- ('rvlimit', '50'),
- ('format', 'json'),
- ]
+ predata = {
+ 'action': 'query',
+ 'prop': 'revisions',
+ 'titles': title.encode('utf-8'),
+ 'rvprop': 'timestamp|user|comment|content',
+ 'rvlimit': '50',
+ }
if username:
- predata.append(('rvexcludeuser', username.encode('utf-8')))
+ predata['rvexcludeuser'] = username.encode('utf-8')
if since:
- predata.append(('rvend', since))
- response, data = self.site.postForm(address, predata)
- data = simplejson.loads(data)
- if 'error' in data:
+ predata['rvend'] = since
+ response, data = query.GetData(predata, self.site, back_response = True)
+ if data.has_key('error'):
raise RuntimeError(data['error'])
page = data['query']['pages'].values()[0]
Modified: trunk/pywikipedia/copyright_clean.py
===================================================================
--- trunk/pywikipedia/copyright_clean.py 2009-08-08 00:42:34 UTC (rev 7130)
+++ trunk/pywikipedia/copyright_clean.py 2009-08-08 01:00:08 UTC (rev 7131)
@@ -9,7 +9,7 @@
# Distributed under the terms of the MIT license.
#
-import httplib, socket, simplejson, re, time
+import httplib, socket, re, time
import config, wikipedia, catlib, pagegenerators, query
from urllib import urlencode
@@ -33,40 +33,22 @@
rev_templateC = re.compile("(?m)^(?:{{/t\|.*?}}\n?)?{{(?:/box|botbox)\|.*?\|(.*?)\|")
def query_api(data):
- predata = [
- ('format', 'json'),
- ('action', 'query'),
- ('prop', 'revisions'),
- data]
- data = urlencode(predata)
- host = wikipedia.getSite().hostname()
- address = wikipedia.getSite().api_address()
- conn = httplib.HTTPConnection(host)
- conn.request("GET", address + data)
- response = conn.getresponse()
- data = response.read()
- conn.close()
- return data
+ predata = {
+ 'action': 'query',
+ 'prop': 'revisions',
+ }
+ predata = query.CombineParams(predata, data)
+ return query.GetData(predata)
def query_old_api(data):
- predata = [
- ('format', 'json'),
- ('what', 'revisions'),
- ('rvlimit', '1'),
- data]
+ predata = {
+ 'what': 'revisions',
+ 'rvlimit': '1',
+ }
+ predata = query.CombineParams(predata, data)
+ return query.GetData(predata, useAPI = False)
- data = urlencode(predata)
- host = wikipedia.getSite().hostname()
- address = wikipedia.getSite().query_address()
- conn = httplib.HTTPConnection(host)
- conn.request("GET", address + data)
- response = conn.getresponse()
- data = response.read()
- conn.close()
-
- return data
-
def old_page_exist(title):
for pageobjs in query_results_titles:
for key in pageobjs['pages']:
@@ -135,9 +117,9 @@
# No more of 50 titles at a time using API
for s in mysplit(query.ListToParam(titles), 50, "|"):
- query_results_titles.append(simplejson.loads(query_api(('titles', s))))
+ query_results_titles.append(query_api({'titles': s,}))
for s in mysplit(query.ListToParam(revids), 50, "|"):
- query_results_revids.append(simplejson.loads(query_api(('revids', s))))
+ query_results_revids.append(query_api({'revids': ,}))
comment_entry = list()
add_separator = False
Modified: trunk/pywikipedia/revertbot.py
===================================================================
--- trunk/pywikipedia/revertbot.py 2009-08-08 00:42:34 UTC (rev 7130)
+++ trunk/pywikipedia/revertbot.py 2009-08-08 01:00:08 UTC (rev 7131)
@@ -1,5 +1,4 @@
-import wikipedia
-import simplejson
+import wikipedia, query
__version__ = '$Id$'
@@ -18,7 +17,6 @@
self.comment = comment
def get_contributions(self, max = -1, ns = None):
- address = self.site.api_address()
predata = {
'action': 'query',
'list': 'usercontribs',
@@ -37,12 +35,11 @@
item = iterator.next()
except StopIteration:
self.log(u'Fetching new batch of contributions')
- response, data = self.site.postForm(address, predata)
- data = simplejson.loads(data)
- if 'error' in data:
+ response, data = query.GetData(predata, self.site, back_response = True)
+ if data.has_key('error'):
raise RuntimeError(data['error'])
- if 'usercontribs' in data.get('query-continue', ()):
- predata.update(data['query-continue']['usercontribs'])
+ if data.has_key('query-continue'):
+ predata['uccontinue'] = data['query-continue']['usercontribs']
else:
never_continue = True
iterator = iter(data['query']['usercontribs'])
@@ -81,10 +78,9 @@
'rvstart': item['timestamp'],
'format': 'json'
}
- response, data = self.site.postForm(self.site.api_address(), predata)
- data = simplejson.loads(data)
+ response, data = query.GetData(predata, self.site, back_response = True)
- if 'error' in data:
+ if data.has_key('error'):
raise RuntimeError(data['error'])
pages = data['query'].get('pages', ())