Pywikipedia-svn October 2009

pywikipedia-svn@lists.wikimedia.org

10 participants
237 discussions

by alexsh＠svn.wikimedia.org

Revision: 7376 Author: alexsh Date: 2009-10-05 16:54:31 +0000 (Mon, 05 Oct 2009) Log Message: ----------- re-organize API verbose Modified Paths: -------------- trunk/pywikipedia/query.py Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2009-10-05 16:53:45 UTC (rev 7375) +++ trunk/pywikipedia/query.py 2009-10-05 16:54:31 UTC (rev 7376) @@ -43,11 +43,29 @@ if not site: site = wikipedia.getSite() + if wikipedia.verbose: + wikipedia.output("====API PARAMS====") for k,v in params.iteritems(): if not IsString(v): params[k] = unicode(v) + if wikipedia.verbose: + if type(v) != int: + if v.count('|') == 0 and len(v) > 40: + wikipedia.output("[%s]: %s (total %d char)" % (k,v[0:30], lev(v)) ) + elif v.count('|') > 8: + wikipedia.output("[%s]: %s (and more %d values)" % (k,v[0:v.index('|')], len(v.split('|')) ) ) + else: + wikipedia.output("[%s]: %s" % (k,v) ) + elif k == u'format': + continue + else: + wikipedia.output("[%s]: %s" % (k,v) ) + if wikipedia.verbose: + wikipedia.output("==================") + - params['format'] = 'json' + if 'format' not in params: + params['format'] = 'json' if not useAPI: params['noprofile'] = '' @@ -59,9 +77,9 @@ # Titles param might be long, case convert it to post request data = None titlecount = 0 - for pLongKey in ['titles', 'pageids', 'ucusers']: # + for pLongKey in ['titles', 'pageids', 'ucusers', 'ususers']: # if pLongKey in params: - titlecount = params[pLongKey].count('|') + titlecount = params[pLongKey].count('|') + 1 if encodeTitle: data = {pLongKey : params[pLongKey]} del params[pLongKey] @@ -80,10 +98,10 @@ path = site.query_address() + urllib.urlencode(params.items()) if wikipedia.verbose: - if titlecount > 0: - wikipedia.output(u"Requesting %d %s from %s:%s" % (titlecount, data.keys()[0], site.lang, path)) + if titlecount > 1: + wikipedia.output(u"Requesting %d %s from %s" % (titlecount, data.keys()[0], site)) else: - wikipedia.output(u"Request %s:%s" % (site.lang, path)) + wikipedia.output(u"Requesting API query from %s" % site) lastError = None retry_idle_time = 1

14 years, 7 months

SVN: [7375] trunk/pywikipedia/wikipedia.py

by alexsh＠svn.wikimedia.org

Revision: 7375 Author: alexsh Date: 2009-10-05 16:53:45 +0000 (Mon, 05 Oct 2009) Log Message: ----------- _load(): API getinfo limit down to 1.12 Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-05 16:52:20 UTC (rev 7374) +++ trunk/pywikipedia/wikipedia.py 2009-10-05 16:53:45 UTC (rev 7375) @@ -1687,7 +1687,7 @@ # just check for HTTP Status 500 (Internal Server Error)? else: output("Unknown Error. API Error code:%s" % data['error']['code'] ) - output("Information:%s" %data['error']['info']) + output("Information:%s" % data['error']['info']) else: if data['edit']['result'] == u"Success": # @@ -1696,7 +1696,7 @@ # if the page update is successed, we need to return code 302 for cheat script who # using status code # - return 302, response.reason, data + return 302, response.reason, data['edit'] solve = self.site().solveCaptcha(data) if solve: @@ -5510,7 +5510,7 @@ # Get data # API Userinfo is available from version 1.11 # preferencetoken available from 1.14 - if config.use_api and self.versionnumber() >= 14: + if config.use_api and self.versionnumber() >= 12: #Query userinfo params = { 'action': 'query',

14 years, 7 months

SVN: [7374] trunk/pywikipedia/redirect.py

by alexsh＠svn.wikimedia.org

Revision: 7374 Author: alexsh Date: 2009-10-05 16:52:20 +0000 (Mon, 05 Oct 2009) Log Message: ----------- fix API override Modified Paths: -------------- trunk/pywikipedia/redirect.py Modified: trunk/pywikipedia/redirect.py =================================================================== --- trunk/pywikipedia/redirect.py 2009-10-04 19:46:54 UTC (rev 7373) +++ trunk/pywikipedia/redirect.py 2009-10-05 16:52:20 UTC (rev 7374) @@ -312,7 +312,7 @@ apiQ = [] for pageid in self.get_redirect_pageids_via_api(number, namespaces, start, until): apiQ.append(pageid) - if len(apiQ) > 500: + if len(apiQ) >= 500: yield apiQ apiQ = [] if apiQ:

14 years, 7 months

SVN: [7373] trunk/pywikipedia

by alexsh＠svn.wikimedia.org

Revision: 7373 Author: alexsh Date: 2009-10-04 19:46:54 +0000 (Sun, 04 Oct 2009) Log Message: ----------- forget: error counting for use params limit directly Modified Paths: -------------- trunk/pywikipedia/userlib.py trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/userlib.py =================================================================== --- trunk/pywikipedia/userlib.py 2009-10-04 19:42:47 UTC (rev 7372) +++ trunk/pywikipedia/userlib.py 2009-10-04 19:46:54 UTC (rev 7373) @@ -217,9 +217,9 @@ for c in result['query']['usercontribs']: yield wikipedia.Page(self.site(), c['title'], defaultNamespace=c['ns']), c['revid'], c['timestamp'], c['comment'] nbresults += 1 - if nbresults >= params['uclimit']: + if nbresults >= limit: break - if 'query-continue' in result and nbresults < params['uclimit']: + if 'query-continue' in result and nbresults < limit: params['ucstart'] = result['query-continue']['usercontribs']['ucstart'] else: break Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-04 19:42:47 UTC (rev 7372) +++ trunk/pywikipedia/wikipedia.py 2009-10-04 19:46:54 UTC (rev 7373) @@ -965,7 +965,7 @@ if count >= tllimit: break - if 'query-continue' in data and count < params['tllimlt']: + if 'query-continue' in data and count < tllimit: params["tlcontinue"] = data["query-continue"]["templates"]["tlcontinue"] else: break @@ -6326,7 +6326,7 @@ if count >= limit: break - if 'query-continue' in data and count < params['eulimit']: + if 'query-continue' in data and count < limit: params['euoffset'] = data[u'query-continue'][u'exturlusage'][u'euoffset'] else: break

14 years, 7 months

SVN: [7372] trunk/pywikipedia

by alexsh＠svn.wikimedia.org

Revision: 7372 Author: alexsh Date: 2009-10-04 19:42:47 +0000 (Sun, 04 Oct 2009) Log Message: ----------- fix queryLimit override Modified Paths: -------------- trunk/pywikipedia/catlib.py trunk/pywikipedia/userlib.py trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/catlib.py =================================================================== --- trunk/pywikipedia/catlib.py 2009-10-04 19:31:30 UTC (rev 7371) +++ trunk/pywikipedia/catlib.py 2009-10-04 19:42:47 UTC (rev 7372) @@ -218,8 +218,10 @@ wikipedia.get_throttle() data = query.GetData(params, self.site()) + count = 0 for memb in data['query']['categorymembers']: + count += 1 # For MediaWiki versions where subcats look like articles if isCatTitle(memb['title'], self.site()): yield SUBCATEGORY, Category(self.site(), memb['title']) @@ -227,8 +229,10 @@ yield ARTICLE, wikipedia.ImagePage(self.site(), memb['title']) else: yield ARTICLE, wikipedia.Page(self.site(), memb['title']) + if count >= params['cmlimit']: + break # try to find a link to the next list page - if 'query-continue' in data: + if 'query-continue' in data and count < params['cmlimilt']: currentPageOffset = data['query-continue']['categorymembers']['cmcontinue'] else: break Modified: trunk/pywikipedia/userlib.py =================================================================== --- trunk/pywikipedia/userlib.py 2009-10-04 19:31:30 UTC (rev 7371) +++ trunk/pywikipedia/userlib.py 2009-10-04 19:42:47 UTC (rev 7372) @@ -217,9 +217,9 @@ for c in result['query']['usercontribs']: yield wikipedia.Page(self.site(), c['title'], defaultNamespace=c['ns']), c['revid'], c['timestamp'], c['comment'] nbresults += 1 - if nbresults >= limit: + if nbresults >= params['uclimit']: break - if 'query-continue' in result: + if 'query-continue' in result and nbresults < params['uclimit']: params['ucstart'] = result['query-continue']['usercontribs']['ucstart'] else: break @@ -266,8 +266,10 @@ for info in data['query']['logevents']: count += 1 yield wikipedia.ImagePage(self.site(), info['title']), info['timestamp'], info['comment'], False + if count >= number: + break - if 'query-continue' in data and count <= number: + if 'query-continue' in data and count < number: params['lestart'] = data['query-continue']['logevents']['lestart'] else: break Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-04 19:31:30 UTC (rev 7371) +++ trunk/pywikipedia/wikipedia.py 2009-10-04 19:42:47 UTC (rev 7372) @@ -965,7 +965,7 @@ if count >= tllimit: break - if 'query-continue' in data: + if 'query-continue' in data and count < params['tllimlt']: params["tlcontinue"] = data["query-continue"]["templates"]["tlcontinue"] else: break @@ -6115,12 +6115,14 @@ get_throttle() data = query.GetData(params, self) - #count = 0 + count = 0 for p in data['query']['allpages']: - #count += 1 + count += 1 yield Page(self, p['title']) + if count >= config.special_page_limit: + break - if 'query-continue' in data: + if 'query-continue' in data and count < params['aplimit']: params['apfrom'] = data['query-continue']['allpages']['apfrom'] else: break @@ -6323,7 +6325,8 @@ yield Page(self, pages['title']) if count >= limit: break - if 'query-continue' in data: + + if 'query-continue' in data and count < params['eulimit']: params['euoffset'] = data[u'query-continue'][u'exturlusage'][u'euoffset'] else: break

14 years, 7 months

SVN: [7371] trunk/pywikipedia/wikipedia.py

by alexsh＠svn.wikimedia.org

Revision: 7371 Author: alexsh Date: 2009-10-04 19:31:30 +0000 (Sun, 04 Oct 2009) Log Message: ----------- typo syntax Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-04 19:29:51 UTC (rev 7370) +++ trunk/pywikipedia/wikipedia.py 2009-10-04 19:31:30 UTC (rev 7371) @@ -956,17 +956,15 @@ while True: data = query.GetData(params, self.site(), encodeTitle = False) pageid = data[u'query'][u'pages'].keys()[0] + for tmp in data['query']['pages'][pageid].values()[0]: + count += 1 + try: + tmpsFound.append(Page(self.site(), tmp['title'], defaultNamespace=tmp['ns']) ) + except TypeError: + pass + if count >= tllimit: + break - for tmp in data['query']['pages'][pageid].values()[0]: - count += 1 - try: - tmpsFound.append(Page(self.site(), tmp['title'], defaultNamespace=tmp['ns']) ) - except TypeError: - pass - - if count >= tllimit: - break - if 'query-continue' in data: params["tlcontinue"] = data["query-continue"]["templates"]["tlcontinue"] else:

14 years, 7 months

SVN: [7370] trunk/pywikipedia/wikipedia.py

by alexsh＠svn.wikimedia.org

Revision: 7370 Author: alexsh Date: 2009-10-04 19:29:51 +0000 (Sun, 04 Oct 2009) Log Message: ----------- fix: all possible queryLimit override Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-04 19:23:38 UTC (rev 7369) +++ trunk/pywikipedia/wikipedia.py 2009-10-04 19:29:51 UTC (rev 7370) @@ -952,14 +952,21 @@ params['tllimit'] = 5000 tmpsFound = [] + count = 0 while True: data = query.GetData(params, self.site(), encodeTitle = False) pageid = data[u'query'][u'pages'].keys()[0] - try: - tmpsFound.extend([Page(self.site(), tmp['title'], defaultNamespace=tmp['ns']) - for tmp in data['query']['pages'][pageid].values()[0] ]) - except TypeError: - pass + + for tmp in data['query']['pages'][pageid].values()[0]: + count += 1 + try: + tmpsFound.append(Page(self.site(), tmp['title'], defaultNamespace=tmp['ns']) ) + except TypeError: + pass + + if count >= tllimit: + break + if 'query-continue' in data: params["tlcontinue"] = data["query-continue"]["templates"]["tlcontinue"] else: @@ -6302,11 +6309,13 @@ 'eulimit': limit, 'euquery': url, } + count = 0 while True: data = query.GetData(params, self) if data['query']['exturlusage'] == []: break for pages in data['query']['exturlusage']: + count += 1 if not siteurl in pages['title']: # the links themselves have similar form if pages['title'] in cache: @@ -6314,6 +6323,8 @@ else: cache.append(pages['title']) yield Page(self, pages['title']) + if count >= limit: + break if 'query-continue' in data: params['euoffset'] = data[u'query-continue'][u'exturlusage'][u'euoffset'] else:

14 years, 7 months

SVN: [7369] trunk/pywikipedia/userlib.py

by alexsh＠svn.wikimedia.org

Revision: 7369 Author: alexsh Date: 2009-10-04 19:23:38 +0000 (Sun, 04 Oct 2009) Log Message: ----------- code typo Modified Paths: -------------- trunk/pywikipedia/userlib.py Modified: trunk/pywikipedia/userlib.py =================================================================== --- trunk/pywikipedia/userlib.py 2009-10-04 19:21:46 UTC (rev 7368) +++ trunk/pywikipedia/userlib.py 2009-10-04 19:23:38 UTC (rev 7369) @@ -208,18 +208,16 @@ params['ucnamespace'] = query.ListToParam(namespace) # An user is likely to contribute on several pages, # keeping track of titles - nbresults=0 + nbresults = 0 while True: - if nbresults>=limit: - break result = query.GetData(params, self.site()) if 'error' in result: wikipedia.output('%s' % result) raise wikipedia.Error for c in result['query']['usercontribs']: yield wikipedia.Page(self.site(), c['title'], defaultNamespace=c['ns']), c['revid'], c['timestamp'], c['comment'] - nbresults=nbresults+1 - if nbresults>=limit: + nbresults += 1 + if nbresults >= limit: break if 'query-continue' in result: params['ucstart'] = result['query-continue']['usercontribs']['ucstart']

14 years, 7 months

SVN: [7368] branches/rewrite/scripts/interwiki_graph.py

by alexsh＠svn.wikimedia.org

Revision: 7368 Author: alexsh Date: 2009-10-04 19:21:46 +0000 (Sun, 04 Oct 2009) Log Message: ----------- interwiki_graph: fix import modules to pywikibot and config2 Modified Paths: -------------- branches/rewrite/scripts/interwiki_graph.py Modified: branches/rewrite/scripts/interwiki_graph.py =================================================================== --- branches/rewrite/scripts/interwiki_graph.py 2009-10-04 18:42:57 UTC (rev 7367) +++ branches/rewrite/scripts/interwiki_graph.py 2009-10-04 19:21:46 UTC (rev 7368) @@ -6,7 +6,8 @@ import pydot except ImportError: pydotfound = False -import wikipedia, config +import pywikibot +from pywikibot import config2 as config # for speedyshare import re

14 years, 7 months

SVN: [7367] trunk/pywikipedia/userlib.py

by siebrand＠svn.wikimedia.org

Revision: 7367 Author: siebrand Date: 2009-10-04 18:42:57 +0000 (Sun, 04 Oct 2009) Log Message: ----------- [ #2872584 ] Get correct number of contributions. Patch contributed by Nakor Wikipedia When asking for user contributions, API always return "query-continue" even if the limit number of contributions has been returned (please see https://bugzilla.wikimedia.org/show_bug.cgi?id=20981 ) This patch solves the pb until the API is fixed. Modified Paths: -------------- trunk/pywikipedia/userlib.py Modified: trunk/pywikipedia/userlib.py =================================================================== --- trunk/pywikipedia/userlib.py 2009-10-04 15:28:18 UTC (rev 7366) +++ trunk/pywikipedia/userlib.py 2009-10-04 18:42:57 UTC (rev 7367) @@ -208,13 +208,19 @@ params['ucnamespace'] = query.ListToParam(namespace) # An user is likely to contribute on several pages, # keeping track of titles + nbresults=0 while True: + if nbresults>=limit: + break result = query.GetData(params, self.site()) if 'error' in result: wikipedia.output('%s' % result) raise wikipedia.Error for c in result['query']['usercontribs']: yield wikipedia.Page(self.site(), c['title'], defaultNamespace=c['ns']), c['revid'], c['timestamp'], c['comment'] + nbresults=nbresults+1 + if nbresults>=limit: + break if 'query-continue' in result: params['ucstart'] = result['query-continue']['usercontribs']['ucstart'] else:

14 years, 7 months

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn October 2009