Pywikipedia-l

pywikipedia-l@lists.wikimedia.org

6393 discussions

SVN: [3942] trunk/pywikipedia
by valhallasw＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3942 Author: valhallasw Date: 2007-08-01 23:31:35 +0000 (Wed, 01 Aug 2007) Log Message: ----------- Changes to allow solve_disambiguation to put asynchronously: * while waiting for input through wikipedia.input(), wikipedia.output() buffers its output and flushes when input has been received * solve_disambiguation now uses put_async Small bugfix: when calling the terminal_interface editor, the ImportError raised when tkinter is not installed is caught. Modified Paths: -------------- trunk/pywikipedia/solve_disambiguation.py trunk/pywikipedia/userinterfaces/terminal_interface.py trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/solve_disambiguation.py =================================================================== --- trunk/pywikipedia/solve_disambiguation.py 2007-08-01 23:17:32 UTC (rev 3941) +++ trunk/pywikipedia/solve_disambiguation.py 2007-08-01 23:31:35 UTC (rev 3942) @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python # -*- coding: utf-8 -*- """ Script to help a human solve disambiguations by presenting a set of options. @@ -75,7 +75,7 @@ import wikipedia, pagegenerators, editarticle # This is a purely interactive robot. We set the delays lower. -wikipedia.put_throttle.setDelay(4) +#wikipedia.put_throttle.setDelay(4) # Summary message when working on disambiguation pages msg = { @@ -523,7 +523,7 @@ if choice in ['y', 'Y']: redir_text = '#%s [[%s]]' % (self.mysite.redirect(default=True), target) try: - refPage.put(redir_text) + refPage.put_async(redir_text) except wikipedia.PageNotSaved, error: wikipedia.output(u'Page not saved: %s' % error.args) else: @@ -716,7 +716,7 @@ wikipedia.output(u'') # save the page try: - refPage.put(text) + refPage.put_async(text) except wikipedia.LockedPage: wikipedia.output(u'Page not saved: page is locked') except wikipedia.PageNotSaved, error: @@ -899,7 +899,10 @@ generator = iter([page]) bot = DisambiguationRobot(always, alternatives, getAlternatives, generator, primary, main_only) - bot.run() + try: + bot.run() + finally: + wikipedia.output(u'\n\nPlease wait for the asynchronous page edits to finish...') if __name__ == "__main__": try: Modified: trunk/pywikipedia/userinterfaces/terminal_interface.py =================================================================== --- trunk/pywikipedia/userinterfaces/terminal_interface.py 2007-08-01 23:17:32 UTC (rev 3941) +++ trunk/pywikipedia/userinterfaces/terminal_interface.py 2007-08-01 23:31:35 UTC (rev 3942) @@ -1,4 +1,4 @@ - + __version__ = '$Id$' import config, transliteration @@ -221,6 +221,10 @@ * jumpIndex - an integer: position at which to put the caret * highlight - a substring; each occurence will be highlighted """ - import gui + try: + import gui + except ImportError, e: + print 'Could not load GUI modules: %s' % e + return text editor = gui.EditBoxWindow() return editor.edit(text, jumpIndex = jumpIndex, highlight = highlight) Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-08-01 23:17:32 UTC (rev 3941) +++ trunk/pywikipedia/wikipedia.py 2007-08-01 23:31:35 UTC (rev 3942) @@ -4454,7 +4454,8 @@ logfile = codecs.open(logfn, 'w', 'utf-8') output_lock = threading.Lock() - +input_lock = threading.Lock() +output_cache = [] def output(text, decoder = None, colors = [], newline = True, toStdout = False): """ Works like print, but uses the encoding used by the user's console @@ -4489,7 +4490,10 @@ # save the text in a logfile (will be written in utf-8) logfile.write(text + '\n') logfile.flush() - ui.output(text, colors = colors, newline = newline, toStdout = toStdout) + if input_lock.locked(): + output_cache.append(((text,), {'colors': colors, 'newline': newline, 'toStdout': toStdout})) + else: + ui.output(text, colors = colors, newline = newline, toStdout = toStdout) finally: output_lock.release() @@ -4506,7 +4510,17 @@ Returns a unicode string. """ - return ui.input(question, colors, password) + input_lock.acquire() + try: + data = ui.input(question, colors, password) + finally: + for output in output_cache: + ui.output(*output[0], **output[1]) + input_lock.release() + for output in output_cache: #for output added between the start of the for loop and the lock release + ui.output(*output[0], **output[1]) + + return data def inputChoice(question, answers, hotkeys, default = None): """

1 0

SVN: [3941] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3941 Author: wikipedian Date: 2007-08-01 23:17:32 +0000 (Wed, 01 Aug 2007) Log Message: ----------- fixed null pointer bug Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 23:10:15 UTC (rev 3940) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 23:17:32 UTC (rev 3941) @@ -273,12 +273,11 @@ if response.status >= 300 and response.status <= 399: #print response.getheaders() redirTarget = response.getheader('Location') - try: - redirTarget.encode('ascii') - except UnicodeError: - redirTarget = unicode(redirTarget, self.getEncodingUsedByServer()) - #print "redirTarget:", redirTarget if redirTarget: + try: + redirTarget.encode('ascii') + except UnicodeError: + redirTarget = redirTarget.decode(self.getEncodingUsedByServer()) if redirTarget.startswith('http://') or redirTarget.startswith('https://'): self.changeUrl(redirTarget) return True

1 0

SVN: [3940] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3940 Author: wikipedian Date: 2007-08-01 23:10:15 +0000 (Wed, 01 Aug 2007) Log Message: ----------- bugfix (parameters wrong) Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 22:50:29 UTC (rev 3939) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 23:10:15 UTC (rev 3940) @@ -322,7 +322,7 @@ # which leads to a cyclic list of redirects. # We simply start from the beginning, but this time, # we don't use HEAD, but GET requests. - redirChecker = LinkChecker(self.redirectChain[0], self.serverEncoding) + redirChecker = LinkChecker(self.redirectChain[0], serverEncoding = self.serverEncoding) return redirChecker.check(useHEAD = False) else: return False, u'HTTP Redirect Loop: %s' % ' -> '.join(self.redirectChain + [self.url]) @@ -332,7 +332,7 @@ # which leads to a long (or infinite) list of redirects. # We simply start from the beginning, but this time, # we don't use HEAD, but GET requests. - redirChecker = LinkChecker(self.redirectChain[0], self.serverEncoding) + redirChecker = LinkChecker(self.redirectChain[0], serverEncoding = self.serverEncoding) return redirChecker.check(useHEAD = False) else: return False, u'Long Chain of Redirects: %s' % ' -> '.join(self.redirectChain + [self.url])

1 0

SVN: [3939] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3939 Author: wikipedian Date: 2007-08-01 22:50:29 +0000 (Wed, 01 Aug 2007) Log Message: ----------- reduce the number of times a page has to be loaded just to find out the server's encoding Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 22:11:21 UTC (rev 3938) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 22:50:29 UTC (rev 3939) @@ -170,12 +170,13 @@ Warning: Also returns false if your Internet connection isn't working correctly! (This will give a Socket Error) ''' - def __init__(self, url, redirectChain = []): + def __init__(self, url, redirectChain = [], serverEncoding = None): """ redirectChain is a list of redirects which were resolved by resolveRedirect(). This is needed to detect redirect loops. """ self.url = url + self.serverEncoding = serverEncoding self.header = { # 'User-agent': wikipedia.useragent, # we fake being Firefox because some webservers block unknown @@ -198,21 +199,34 @@ return httplib.HTTPSConnection(self.host) def getEncodingUsedByServer(self): - # TODO: We could maybe save a few accesses here by caching. - try: - conn = self.getConnection() - conn.request('HEAD', '/', None, self.header) - response = conn.getresponse() + if not self.serverEncoding: + try: + print conn.__dict__ + wikipedia.output(u'Contacting server %s to find out its default encoding...' % self.conn) + conn = self.getConnection() + conn.request('HEAD', '/', None, self.header) + response = conn.getresponse() - ct = response.getheader('Content-Type') - charsetR = re.compile('charset=(.+)') - charset = charsetR.search(ct).group(1) - return charset - except: - wikipedia.output(u'Error retrieving server\'s default charset. Using ISO 8859-1.') - # most browsers use ISO 8859-1 (Latin-1) as the default. - return 'iso8859-1' + self.readEncodingFromResponse() + except: + pass + if not self.serverEncoding: + # TODO: We might also load a page, then check for an encoding + # definition in a HTML meta tag. + wikipedia.output(u'Error retrieving server\'s default charset. Using ISO 8859-1.') + # most browsers use ISO 8859-1 (Latin-1) as the default. + self.serverEncoding = 'iso8859-1' + return self.serverEncoding + def readEncodingFromResponse(self, response): + if not self.serverEncoding: + try: + ct = response.getheader('Content-Type') + charsetR = re.compile('charset=(.+)') + charset = charsetR.search(ct).group(1) + self.serverEncoding = charset + except: + pass def changeUrl(self, url): self.url = url @@ -229,7 +243,6 @@ self.path.encode('ascii') self.query.encode('ascii') except UnicodeEncodeError: - wikipedia.output(u'%s contains non-ASCII characters. Contacting server to find out its default encoding...' % self.url) encoding = self.getEncodingUsedByServer() self.path = unicode(urllib.quote(self.path.encode(encoding))) self.query = unicode(urllib.quote(self.query.encode(encoding), '=&')) @@ -249,6 +262,8 @@ else: conn.request('GET', '%s%s' % (self.path, self.query), None, self.header) response = conn.getresponse() + # read the server's encoding, in case we need it later + self.readEncodingFromResponse(response) except httplib.BadStatusLine: # Some servers don't seem to handle HEAD requests properly, # e.g. http://www.radiorus.ru/ which is running on a very old @@ -258,7 +273,10 @@ if response.status >= 300 and response.status <= 399: #print response.getheaders() redirTarget = response.getheader('Location') - redirTarget = unicode(redirTarget, self.getEncodingUsedByServer()) + try: + redirTarget.encode('ascii') + except UnicodeError: + redirTarget = unicode(redirTarget, self.getEncodingUsedByServer()) #print "redirTarget:", redirTarget if redirTarget: if redirTarget.startswith('http://') or redirTarget.startswith('https://'): @@ -283,7 +301,7 @@ return True else: return False # not a redirect - + def check(self, useHEAD = True): """ Returns True and the server status message if the page is alive. @@ -304,7 +322,7 @@ # which leads to a cyclic list of redirects. # We simply start from the beginning, but this time, # we don't use HEAD, but GET requests. - redirChecker = LinkChecker(self.redirectChain[0]) + redirChecker = LinkChecker(self.redirectChain[0], self.serverEncoding) return redirChecker.check(useHEAD = False) else: return False, u'HTTP Redirect Loop: %s' % ' -> '.join(self.redirectChain + [self.url]) @@ -314,12 +332,12 @@ # which leads to a long (or infinite) list of redirects. # We simply start from the beginning, but this time, # we don't use HEAD, but GET requests. - redirChecker = LinkChecker(self.redirectChain[0]) + redirChecker = LinkChecker(self.redirectChain[0], self.serverEncoding) return redirChecker.check(useHEAD = False) else: return False, u'Long Chain of Redirects: %s' % ' -> '.join(self.redirectChain + [self.url]) else: - redirChecker = LinkChecker(self.url, self.redirectChain) + redirChecker = LinkChecker(self.url, self.redirectChain, self.serverEncoding) return redirChecker.check(useHEAD = useHEAD) else: try: @@ -336,7 +354,8 @@ response = conn.getresponse() except Exception, arg: return False, u'Error: %s' % arg - #wikipedia.output('%s: %s' % (self.url, response.status)) + # read the server's encoding, in case we need it later + self.readEncodingFromResponse(response) # site down if the server status is between 400 and 499 siteDown = response.status in range(400, 500) return not siteDown, '%s %s' % (response.status, response.reason)

1 0

SVN: [3938] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3938 Author: wikipedian Date: 2007-08-01 22:11:21 +0000 (Wed, 01 Aug 2007) Log Message: ----------- decode HTTP redirect targets use Latin-1 as default server encoding instead of UTF-8 Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 21:53:08 UTC (rev 3937) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 22:11:21 UTC (rev 3938) @@ -198,6 +198,7 @@ return httplib.HTTPSConnection(self.host) def getEncodingUsedByServer(self): + # TODO: We could maybe save a few accesses here by caching. try: conn = self.getConnection() conn.request('HEAD', '/', None, self.header) @@ -208,8 +209,9 @@ charset = charsetR.search(ct).group(1) return charset except: - wikipedia.output(u'Error retrieving server\'s default charset. Using UTF-8.') - return 'utf-8' + wikipedia.output(u'Error retrieving server\'s default charset. Using ISO 8859-1.') + # most browsers use ISO 8859-1 (Latin-1) as the default. + return 'iso8859-1' def changeUrl(self, url): @@ -256,13 +258,14 @@ if response.status >= 300 and response.status <= 399: #print response.getheaders() redirTarget = response.getheader('Location') + redirTarget = unicode(redirTarget, self.getEncodingUsedByServer()) #print "redirTarget:", redirTarget if redirTarget: if redirTarget.startswith('http://') or redirTarget.startswith('https://'): self.changeUrl(redirTarget) return True elif redirTarget.startswith('/'): - self.changeUrl('%s://%s%s' % (self.protocol, self.host, redirTarget)) + self.changeUrl(u'%s://%s%s' % (self.protocol, self.host, redirTarget)) return True else: # redirect to relative position # cut off filename

1 0

SVN: [3937] trunk/pywikipedia/wikipedia.py
by valhallasw＠svn.wikimedia.org 01 Aug '07

01 Aug '07

Revision: 3937 Author: valhallasw Date: 2007-08-01 21:53:08 +0000 (Wed, 01 Aug 2007) Log Message: ----------- Added version output when '-v' is used; added version output and doctest when wikipedia.py is called Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-08-01 21:37:41 UTC (rev 3936) +++ trunk/pywikipedia/wikipedia.py 2007-08-01 21:53:08 UTC (rev 3937) @@ -4203,6 +4203,9 @@ global logfile logfile = None elif arg == '-verbose' or arg == "-v": + import version + output('Pywikipediabot %s' % (version.getversion())) + output('Python %s' % (sys.version)) verbose += 1 else: # the argument is not global. Let the specific bot script care @@ -4648,3 +4651,11 @@ authhandler = urllib2.HTTPBasicAuthHandler(passman) authenticateURLopener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),authhandler) urllib2.install_opener(authenticateURLopener) + +if __name__ == '__main__': + import version, doctest + print 'Pywikipediabot %s' % version.getversion() + print 'Python %s' % sys.version + doctest.testmod() + + \ No newline at end of file

1 0

SVN: [3936] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 01 Aug '07

01 Aug '07

Revision: 3936 Author: wikipedian Date: 2007-08-01 21:37:41 +0000 (Wed, 01 Aug 2007) Log Message: ----------- some servers redirect to .. although we are already in the root directory; ignore this. Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 21:19:26 UTC (rev 3935) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 21:37:41 UTC (rev 3936) @@ -270,9 +270,12 @@ # handle redirect to parent directory while redirTarget.startswith('../'): redirTarget = redirTarget[3:] - # change /foo/bar/ to /foo/ - directory = directory[:-1] - directory = directory[:directory.rindex('/') + 1] + # some servers redirect to .. although we are already + # in the root directory; ignore this. + if directory != '/': + # change /foo/bar/ to /foo/ + directory = directory[:-1] + directory = directory[:directory.rindex('/') + 1] self.changeUrl('%s://%s%s%s' % (self.protocol, self.host, directory, redirTarget)) return True else:

1 0

SVN: [3935] trunk/pywikipedia/version.py
by valhallasw＠svn.wikimedia.org 01 Aug '07

01 Aug '07

Revision: 3935 Author: valhallasw Date: 2007-08-01 21:19:26 +0000 (Wed, 01 Aug 2007) Log Message: ----------- New: version.py. returns pywikipedia version based on (in order) .svn/entries, version (file for use in nightlies) and wikipedia.__version__. Added Paths: ----------- trunk/pywikipedia/version.py Added: trunk/pywikipedia/version.py =================================================================== --- trunk/pywikipedia/version.py (rev 0) +++ trunk/pywikipedia/version.py 2007-08-01 21:19:26 UTC (rev 3935) @@ -0,0 +1,56 @@ +""" Module to determine the pywikipedia version (tag, revision and date) """ +# +# (C) Merlijn 'valhallasw' van Deen +# +# Distributed under the terms of the MIT license. +# +__version__ = '$Id$' + +import wikipediatools +import time +import sys + +class ParseError(Exception): + """ Parsing went wrong """ + +def getversion(): + try: + (tag, rev, date) = getversion_svn() + except Exception, e: + try: + (tag, rev, date) = getversion_nightly() + except IOError, e: + import wikipedia + d = wikipedia.__version__.split(' ') + tag = '' + date = time.strptime('T'.join(d[3:5]), '%Y-%m-%dT%H:%M:%SZ') + rev = d[2] + ' (wikipedia.py)' + + datestring = time.strftime('%b %d %Y, %H:%M:%S', date) + return '%s (r%s, %s)' % (tag, rev, datestring) + +def getversion_svn(): + entries = open(wikipediatools.absoluteFilename('.svn/entries')) + for i in range(4): + entries.readline() + tag = entries.readline().replace('svn+ssh://svn.wikimedia.org/svnroot/pywikipedia/', '').strip() + for i in range(4): + entries.readline() + date = time.strptime(entries.readline()[:19],'%Y-%m-%dT%H:%M:%S') + rev = entries.readline()[:-1] + if not date or not tag or not rev: + raise ParseError + return (tag, rev, date) + +def getversion_nightly(): + data = open(wikipediatools.absoluteFilename('version')) + tag = data.readline().strip() + date = time.strptime(data.readline()[:19],'%Y-%m-%dT%H:%M:%S') + rev = data.readline().strip() + if not date or not tag or not rev: + raise ParseError + return (tag, rev, date) + +if __name__ == '__main__': + print 'Pywikipedia %s' % getversion() + print 'Python %s' % sys.version \ No newline at end of file Property changes on: trunk/pywikipedia/version.py ___________________________________________________________________ Name: svn:keywords + Id *.c = svn:eol-style=native *.cpp = svn:eol-style=native *.h = svn:eol-style=native *.dsp = svn:eol-style=CRLF *.dsw = svn:eol-style=CRLF *.sh = svn:eol-style=native Name: svn:executable *.txt + svn:eol-style=native *.png = svn:mime-type=image/png *.jpg = svn:mime-type=image/jpeg Makefile = svn:eol-style=native Name: svn:eol-style + native

1 0

SVN: [3934] trunk/pywikipedia/wikipedia.py
by valhallasw＠svn.wikimedia.org 01 Aug '07

01 Aug '07

Revision: 3934 Author: valhallasw Date: 2007-08-01 19:53:05 +0000 (Wed, 01 Aug 2007) Log Message: ----------- Changed Site('de', 'wikipedia') comparison to 'wikipedia:de'. The latter is fine with wikipedia_family.py missing, the first is not. Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-08-01 16:37:37 UTC (rev 3933) +++ trunk/pywikipedia/wikipedia.py 2007-08-01 19:53:05 UTC (rev 3934) @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- """ Library to get and put pages on a MediaWiki. @@ -2850,7 +2850,7 @@ if site is None: site = getSite() - if site == Site('de', 'wikipedia'): + if site.sitename() == 'wikipedia:de': raise Error('The PyWikipediaBot is no longer allowed to touch categories on the German Wikipedia. See http://de.wikipedia.org/wiki/Hilfe_Diskussion:Personendaten/Archiv2#Positio…') s = categoryFormat(new, insite = site)

1 0

SVN: [3933] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 01 Aug '07

01 Aug '07

Revision: 3933 Author: wikipedian Date: 2007-08-01 16:37:37 +0000 (Wed, 01 Aug 2007) Log Message: ----------- fixed handling of URLs in multi-line templates, e.g. the infobox in http://de.wikipedia.org/wiki/Camon_%28Ari%C3%A8ge%29 Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 16:24:02 UTC (rev 3932) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 16:37:37 UTC (rev 3933) @@ -107,7 +107,7 @@ # might be a | or a } directly after a URL which does not belong to # the URL itself. # Blow up templates with spaces to avoid these problems. - templateWithParamsR = re.compile(r'{{(.*?[^ ])\|([^ ].*?)}}') + templateWithParamsR = re.compile(r'{{(.*?[^ ])\|([^ ].*?)}}', re.DOTALL) while templateWithParamsR.search(text): text = templateWithParamsR.sub(r'{{ \1 | \2 }}', text)

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

Pywikipedia-l