pywikibot

pywikibot@lists.wikimedia.org

1 participants
6836 discussions

[Pywikipedia-l] Subversion
by Andre Engels 02 Aug '07

02 Aug '07

So how do I get the new versions from Subversion? And how do I put them? -- Andre Engels, andreengels(a)gmail.com ICQ: 6260644 -- Skype: a_engels

1 0

[Pywikipedia-l] SVN: [3944] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3944 Author: wikipedian Date: 2007-08-01 23:41:57 +0000 (Wed, 01 Aug 2007) Log Message: ----------- prevented an infinite loop when the server keeps giving httplib.BadStatusLine exceptions Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 23:34:27 UTC (rev 3943) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 23:41:57 UTC (rev 3944) @@ -269,7 +269,10 @@ # e.g. http://www.radiorus.ru/ which is running on a very old # Apache server. Using GET instead works on these (but it uses # more bandwidth). - return self.resolveRedirect(useHEAD = False) + if useHEAD: + return self.resolveRedirect(useHEAD = False) + else: + raise if response.status >= 300 and response.status <= 399: #print response.getheaders() redirTarget = response.getheader('Location')

1 0

[Pywikipedia-l] SVN: [3943] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3943 Author: wikipedian Date: 2007-08-01 23:34:27 +0000 (Wed, 01 Aug 2007) Log Message: ----------- report URLs with invalid character data, and those that are too long to be processed by the IDNA library Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 23:31:35 UTC (rev 3942) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 23:34:27 UTC (rev 3943) @@ -308,6 +308,8 @@ """ try: wasRedirected = self.resolveRedirect(useHEAD = useHEAD) + except UnicodeError, arg: + return False, u'Encoding Error: %s' % arg except httplib.error, arg: return False, u'HTTP Error: %s' % arg except socket.error, arg:

1 0

[Pywikipedia-l] SVN: [3942] trunk/pywikipedia
by valhallasw＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3942 Author: valhallasw Date: 2007-08-01 23:31:35 +0000 (Wed, 01 Aug 2007) Log Message: ----------- Changes to allow solve_disambiguation to put asynchronously: * while waiting for input through wikipedia.input(), wikipedia.output() buffers its output and flushes when input has been received * solve_disambiguation now uses put_async Small bugfix: when calling the terminal_interface editor, the ImportError raised when tkinter is not installed is caught. Modified Paths: -------------- trunk/pywikipedia/solve_disambiguation.py trunk/pywikipedia/userinterfaces/terminal_interface.py trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/solve_disambiguation.py =================================================================== --- trunk/pywikipedia/solve_disambiguation.py 2007-08-01 23:17:32 UTC (rev 3941) +++ trunk/pywikipedia/solve_disambiguation.py 2007-08-01 23:31:35 UTC (rev 3942) @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/python # -*- coding: utf-8 -*- """ Script to help a human solve disambiguations by presenting a set of options. @@ -75,7 +75,7 @@ import wikipedia, pagegenerators, editarticle # This is a purely interactive robot. We set the delays lower. -wikipedia.put_throttle.setDelay(4) +#wikipedia.put_throttle.setDelay(4) # Summary message when working on disambiguation pages msg = { @@ -523,7 +523,7 @@ if choice in ['y', 'Y']: redir_text = '#%s [[%s]]' % (self.mysite.redirect(default=True), target) try: - refPage.put(redir_text) + refPage.put_async(redir_text) except wikipedia.PageNotSaved, error: wikipedia.output(u'Page not saved: %s' % error.args) else: @@ -716,7 +716,7 @@ wikipedia.output(u'') # save the page try: - refPage.put(text) + refPage.put_async(text) except wikipedia.LockedPage: wikipedia.output(u'Page not saved: page is locked') except wikipedia.PageNotSaved, error: @@ -899,7 +899,10 @@ generator = iter([page]) bot = DisambiguationRobot(always, alternatives, getAlternatives, generator, primary, main_only) - bot.run() + try: + bot.run() + finally: + wikipedia.output(u'\n\nPlease wait for the asynchronous page edits to finish...') if __name__ == "__main__": try: Modified: trunk/pywikipedia/userinterfaces/terminal_interface.py =================================================================== --- trunk/pywikipedia/userinterfaces/terminal_interface.py 2007-08-01 23:17:32 UTC (rev 3941) +++ trunk/pywikipedia/userinterfaces/terminal_interface.py 2007-08-01 23:31:35 UTC (rev 3942) @@ -1,4 +1,4 @@ - + __version__ = '$Id$' import config, transliteration @@ -221,6 +221,10 @@ * jumpIndex - an integer: position at which to put the caret * highlight - a substring; each occurence will be highlighted """ - import gui + try: + import gui + except ImportError, e: + print 'Could not load GUI modules: %s' % e + return text editor = gui.EditBoxWindow() return editor.edit(text, jumpIndex = jumpIndex, highlight = highlight) Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-08-01 23:17:32 UTC (rev 3941) +++ trunk/pywikipedia/wikipedia.py 2007-08-01 23:31:35 UTC (rev 3942) @@ -4454,7 +4454,8 @@ logfile = codecs.open(logfn, 'w', 'utf-8') output_lock = threading.Lock() - +input_lock = threading.Lock() +output_cache = [] def output(text, decoder = None, colors = [], newline = True, toStdout = False): """ Works like print, but uses the encoding used by the user's console @@ -4489,7 +4490,10 @@ # save the text in a logfile (will be written in utf-8) logfile.write(text + '\n') logfile.flush() - ui.output(text, colors = colors, newline = newline, toStdout = toStdout) + if input_lock.locked(): + output_cache.append(((text,), {'colors': colors, 'newline': newline, 'toStdout': toStdout})) + else: + ui.output(text, colors = colors, newline = newline, toStdout = toStdout) finally: output_lock.release() @@ -4506,7 +4510,17 @@ Returns a unicode string. """ - return ui.input(question, colors, password) + input_lock.acquire() + try: + data = ui.input(question, colors, password) + finally: + for output in output_cache: + ui.output(*output[0], **output[1]) + input_lock.release() + for output in output_cache: #for output added between the start of the for loop and the lock release + ui.output(*output[0], **output[1]) + + return data def inputChoice(question, answers, hotkeys, default = None): """

1 0

[Pywikipedia-l] SVN: [3941] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3941 Author: wikipedian Date: 2007-08-01 23:17:32 +0000 (Wed, 01 Aug 2007) Log Message: ----------- fixed null pointer bug Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 23:10:15 UTC (rev 3940) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 23:17:32 UTC (rev 3941) @@ -273,12 +273,11 @@ if response.status >= 300 and response.status <= 399: #print response.getheaders() redirTarget = response.getheader('Location') - try: - redirTarget.encode('ascii') - except UnicodeError: - redirTarget = unicode(redirTarget, self.getEncodingUsedByServer()) - #print "redirTarget:", redirTarget if redirTarget: + try: + redirTarget.encode('ascii') + except UnicodeError: + redirTarget = redirTarget.decode(self.getEncodingUsedByServer()) if redirTarget.startswith('http://') or redirTarget.startswith('https://'): self.changeUrl(redirTarget) return True

1 0

[Pywikipedia-l] SVN: [3940] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3940 Author: wikipedian Date: 2007-08-01 23:10:15 +0000 (Wed, 01 Aug 2007) Log Message: ----------- bugfix (parameters wrong) Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 22:50:29 UTC (rev 3939) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 23:10:15 UTC (rev 3940) @@ -322,7 +322,7 @@ # which leads to a cyclic list of redirects. # We simply start from the beginning, but this time, # we don't use HEAD, but GET requests. - redirChecker = LinkChecker(self.redirectChain[0], self.serverEncoding) + redirChecker = LinkChecker(self.redirectChain[0], serverEncoding = self.serverEncoding) return redirChecker.check(useHEAD = False) else: return False, u'HTTP Redirect Loop: %s' % ' -> '.join(self.redirectChain + [self.url]) @@ -332,7 +332,7 @@ # which leads to a long (or infinite) list of redirects. # We simply start from the beginning, but this time, # we don't use HEAD, but GET requests. - redirChecker = LinkChecker(self.redirectChain[0], self.serverEncoding) + redirChecker = LinkChecker(self.redirectChain[0], serverEncoding = self.serverEncoding) return redirChecker.check(useHEAD = False) else: return False, u'Long Chain of Redirects: %s' % ' -> '.join(self.redirectChain + [self.url])

1 0

[Pywikipedia-l] SVN: [3939] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3939 Author: wikipedian Date: 2007-08-01 22:50:29 +0000 (Wed, 01 Aug 2007) Log Message: ----------- reduce the number of times a page has to be loaded just to find out the server's encoding Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 22:11:21 UTC (rev 3938) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 22:50:29 UTC (rev 3939) @@ -170,12 +170,13 @@ Warning: Also returns false if your Internet connection isn't working correctly! (This will give a Socket Error) ''' - def __init__(self, url, redirectChain = []): + def __init__(self, url, redirectChain = [], serverEncoding = None): """ redirectChain is a list of redirects which were resolved by resolveRedirect(). This is needed to detect redirect loops. """ self.url = url + self.serverEncoding = serverEncoding self.header = { # 'User-agent': wikipedia.useragent, # we fake being Firefox because some webservers block unknown @@ -198,21 +199,34 @@ return httplib.HTTPSConnection(self.host) def getEncodingUsedByServer(self): - # TODO: We could maybe save a few accesses here by caching. - try: - conn = self.getConnection() - conn.request('HEAD', '/', None, self.header) - response = conn.getresponse() + if not self.serverEncoding: + try: + print conn.__dict__ + wikipedia.output(u'Contacting server %s to find out its default encoding...' % self.conn) + conn = self.getConnection() + conn.request('HEAD', '/', None, self.header) + response = conn.getresponse() - ct = response.getheader('Content-Type') - charsetR = re.compile('charset=(.+)') - charset = charsetR.search(ct).group(1) - return charset - except: - wikipedia.output(u'Error retrieving server\'s default charset. Using ISO 8859-1.') - # most browsers use ISO 8859-1 (Latin-1) as the default. - return 'iso8859-1' + self.readEncodingFromResponse() + except: + pass + if not self.serverEncoding: + # TODO: We might also load a page, then check for an encoding + # definition in a HTML meta tag. + wikipedia.output(u'Error retrieving server\'s default charset. Using ISO 8859-1.') + # most browsers use ISO 8859-1 (Latin-1) as the default. + self.serverEncoding = 'iso8859-1' + return self.serverEncoding + def readEncodingFromResponse(self, response): + if not self.serverEncoding: + try: + ct = response.getheader('Content-Type') + charsetR = re.compile('charset=(.+)') + charset = charsetR.search(ct).group(1) + self.serverEncoding = charset + except: + pass def changeUrl(self, url): self.url = url @@ -229,7 +243,6 @@ self.path.encode('ascii') self.query.encode('ascii') except UnicodeEncodeError: - wikipedia.output(u'%s contains non-ASCII characters. Contacting server to find out its default encoding...' % self.url) encoding = self.getEncodingUsedByServer() self.path = unicode(urllib.quote(self.path.encode(encoding))) self.query = unicode(urllib.quote(self.query.encode(encoding), '=&')) @@ -249,6 +262,8 @@ else: conn.request('GET', '%s%s' % (self.path, self.query), None, self.header) response = conn.getresponse() + # read the server's encoding, in case we need it later + self.readEncodingFromResponse(response) except httplib.BadStatusLine: # Some servers don't seem to handle HEAD requests properly, # e.g. http://www.radiorus.ru/ which is running on a very old @@ -258,7 +273,10 @@ if response.status >= 300 and response.status <= 399: #print response.getheaders() redirTarget = response.getheader('Location') - redirTarget = unicode(redirTarget, self.getEncodingUsedByServer()) + try: + redirTarget.encode('ascii') + except UnicodeError: + redirTarget = unicode(redirTarget, self.getEncodingUsedByServer()) #print "redirTarget:", redirTarget if redirTarget: if redirTarget.startswith('http://') or redirTarget.startswith('https://'): @@ -283,7 +301,7 @@ return True else: return False # not a redirect - + def check(self, useHEAD = True): """ Returns True and the server status message if the page is alive. @@ -304,7 +322,7 @@ # which leads to a cyclic list of redirects. # We simply start from the beginning, but this time, # we don't use HEAD, but GET requests. - redirChecker = LinkChecker(self.redirectChain[0]) + redirChecker = LinkChecker(self.redirectChain[0], self.serverEncoding) return redirChecker.check(useHEAD = False) else: return False, u'HTTP Redirect Loop: %s' % ' -> '.join(self.redirectChain + [self.url]) @@ -314,12 +332,12 @@ # which leads to a long (or infinite) list of redirects. # We simply start from the beginning, but this time, # we don't use HEAD, but GET requests. - redirChecker = LinkChecker(self.redirectChain[0]) + redirChecker = LinkChecker(self.redirectChain[0], self.serverEncoding) return redirChecker.check(useHEAD = False) else: return False, u'Long Chain of Redirects: %s' % ' -> '.join(self.redirectChain + [self.url]) else: - redirChecker = LinkChecker(self.url, self.redirectChain) + redirChecker = LinkChecker(self.url, self.redirectChain, self.serverEncoding) return redirChecker.check(useHEAD = useHEAD) else: try: @@ -336,7 +354,8 @@ response = conn.getresponse() except Exception, arg: return False, u'Error: %s' % arg - #wikipedia.output('%s: %s' % (self.url, response.status)) + # read the server's encoding, in case we need it later + self.readEncodingFromResponse(response) # site down if the server status is between 400 and 499 siteDown = response.status in range(400, 500) return not siteDown, '%s %s' % (response.status, response.reason)

1 0

[Pywikipedia-l] SVN: [3938] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3938 Author: wikipedian Date: 2007-08-01 22:11:21 +0000 (Wed, 01 Aug 2007) Log Message: ----------- decode HTTP redirect targets use Latin-1 as default server encoding instead of UTF-8 Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 21:53:08 UTC (rev 3937) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 22:11:21 UTC (rev 3938) @@ -198,6 +198,7 @@ return httplib.HTTPSConnection(self.host) def getEncodingUsedByServer(self): + # TODO: We could maybe save a few accesses here by caching. try: conn = self.getConnection() conn.request('HEAD', '/', None, self.header) @@ -208,8 +209,9 @@ charset = charsetR.search(ct).group(1) return charset except: - wikipedia.output(u'Error retrieving server\'s default charset. Using UTF-8.') - return 'utf-8' + wikipedia.output(u'Error retrieving server\'s default charset. Using ISO 8859-1.') + # most browsers use ISO 8859-1 (Latin-1) as the default. + return 'iso8859-1' def changeUrl(self, url): @@ -256,13 +258,14 @@ if response.status >= 300 and response.status <= 399: #print response.getheaders() redirTarget = response.getheader('Location') + redirTarget = unicode(redirTarget, self.getEncodingUsedByServer()) #print "redirTarget:", redirTarget if redirTarget: if redirTarget.startswith('http://') or redirTarget.startswith('https://'): self.changeUrl(redirTarget) return True elif redirTarget.startswith('/'): - self.changeUrl('%s://%s%s' % (self.protocol, self.host, redirTarget)) + self.changeUrl(u'%s://%s%s' % (self.protocol, self.host, redirTarget)) return True else: # redirect to relative position # cut off filename

1 0

[Pywikipedia-l] SVN: [3937] trunk/pywikipedia/wikipedia.py
by valhallasw＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3937 Author: valhallasw Date: 2007-08-01 21:53:08 +0000 (Wed, 01 Aug 2007) Log Message: ----------- Added version output when '-v' is used; added version output and doctest when wikipedia.py is called Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-08-01 21:37:41 UTC (rev 3936) +++ trunk/pywikipedia/wikipedia.py 2007-08-01 21:53:08 UTC (rev 3937) @@ -4203,6 +4203,9 @@ global logfile logfile = None elif arg == '-verbose' or arg == "-v": + import version + output('Pywikipediabot %s' % (version.getversion())) + output('Python %s' % (sys.version)) verbose += 1 else: # the argument is not global. Let the specific bot script care @@ -4648,3 +4651,11 @@ authhandler = urllib2.HTTPBasicAuthHandler(passman) authenticateURLopener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),authhandler) urllib2.install_opener(authenticateURLopener) + +if __name__ == '__main__': + import version, doctest + print 'Pywikipediabot %s' % version.getversion() + print 'Python %s' % sys.version + doctest.testmod() + + \ No newline at end of file

1 0

[Pywikipedia-l] SVN: [3936] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 02 Aug '07

02 Aug '07

Revision: 3936 Author: wikipedian Date: 2007-08-01 21:37:41 +0000 (Wed, 01 Aug 2007) Log Message: ----------- some servers redirect to .. although we are already in the root directory; ignore this. Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 21:19:26 UTC (rev 3935) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 21:37:41 UTC (rev 3936) @@ -270,9 +270,12 @@ # handle redirect to parent directory while redirTarget.startswith('../'): redirTarget = redirTarget[3:] - # change /foo/bar/ to /foo/ - directory = directory[:-1] - directory = directory[:directory.rindex('/') + 1] + # some servers redirect to .. although we are already + # in the root directory; ignore this. + if directory != '/': + # change /foo/bar/ to /foo/ + directory = directory[:-1] + directory = directory[:directory.rindex('/') + 1] self.changeUrl('%s://%s%s%s' % (self.protocol, self.host, directory, redirTarget)) return True else:

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

pywikibot