Revision: 7522 Author: alexsh Date: 2009-10-21 02:31:09 +0000 (Wed, 21 Oct 2009)
Log Message: ----------- revert...not compatible for python 2.5 Orz
Modified Paths: -------------- trunk/pywikipedia/interwiki.py trunk/pywikipedia/login.py trunk/pywikipedia/query.py trunk/pywikipedia/upload.py trunk/pywikipedia/userlib.py trunk/pywikipedia/weblinkchecker.py trunk/pywikipedia/wikipedia.py
Property Changed: ---------------- trunk/pywikipedia/commons_category_redirect.py trunk/pywikipedia/maintenance/cleanident
Property changes on: trunk/pywikipedia/commons_category_redirect.py ___________________________________________________________________ Deleted: svn:mergeinfo -
Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-10-21 02:12:58 UTC (rev 7521) +++ trunk/pywikipedia/interwiki.py 2009-10-21 02:31:09 UTC (rev 7522) @@ -1596,7 +1596,7 @@ timeout=60 while 1: try: - status, data = page.put(newtext, comment = mcomment) + status, reason, data = page.put(newtext, comment = mcomment) except wikipedia.LockedPage: wikipedia.output(u'Page %s is locked. Skipping.' % (page.title(),)) raise SaveError @@ -1628,7 +1628,7 @@ if str(status) == '302': return True else: - wikipedia.output(u'%s' % status) + wikipedia.output(u'%s %s' % (status, reason)) return False elif answer == 'g': raise GiveUpOnPage
Modified: trunk/pywikipedia/login.py =================================================================== --- trunk/pywikipedia/login.py 2009-10-21 02:12:58 UTC (rev 7521) +++ trunk/pywikipedia/login.py 2009-10-21 02:31:09 UTC (rev 7522) @@ -182,7 +182,7 @@ fakepredata['wpPassword'] = fakepredata['lgpassword'] = u'XXXXX' wikipedia.output(u"self.site.postData(%s, %s)" % (address, self.site.urlEncode(fakepredata))) fakeresponsemsg = re.sub(r"(session|Token)=..........", r"session=XXXXXXXXXX", response.msg.__str__()) - wikipedia.output(u"%s\n%s" % (response.code, fakeresponsemsg)) + wikipedia.output(u"%s/%s\n%s" % (response.status, response.reason, fakeresponsemsg)) wikipedia.output(u"%s" % data) Reat=re.compile(': (.*?);') L = []
Property changes on: trunk/pywikipedia/maintenance/cleanident ___________________________________________________________________ Deleted: svn:mergeinfo -
Modified: trunk/pywikipedia/query.py =================================================================== --- trunk/pywikipedia/query.py 2009-10-21 02:12:58 UTC (rev 7521) +++ trunk/pywikipedia/query.py 2009-10-21 02:31:09 UTC (rev 7522) @@ -115,6 +115,11 @@ (('file', params['filename'].encode(site.encoding()), cont),), site.cookies(sysop=sysop) ) + elif site.hostname() in wikipedia.config.authenticate.keys(): + params["Content-type"] = "application/x-www-form-urlencoded" + params["User-agent"] = useragent + res = urllib2.urlopen(urllib2.Request(site.protocol() + '://' + site.hostname() + address, site.urlEncode(params))) + jsontext = res.read() elif params['action'] in postAC: res, jsontext = site.postForm(path, params, sysop, site.cookies(sysop = sysop) ) else:
Modified: trunk/pywikipedia/upload.py =================================================================== --- trunk/pywikipedia/upload.py 2009-10-21 02:12:58 UTC (rev 7521) +++ trunk/pywikipedia/upload.py 2009-10-21 02:31:09 UTC (rev 7522) @@ -347,14 +347,14 @@ # ATTENTION: if you changed your Wikimedia Commons account not to show # an English interface, this detection will fail! success_msg = self.targetSite.mediawiki_message('successfulupload') - if success_msg in returned_html or response.code == 302: + if success_msg in returned_html or response.status == 302: wikipedia.output(u"Upload successful.") # The following is not a good idea, because the server also gives a 200 when # something went wrong. - #if response.code in [200, 302]: + #if response.status in [200, 302]: # wikipedia.output(u"Upload successful.")
- elif response.code == 301: + elif response.status == 301: wikipedia.output(u"Following redirect...") address = response.getheader('Location') wikipedia.output(u"Changed upload address to %s. Please update %s.py" % (address, self.targetSite.family.__module__)) @@ -368,7 +368,7 @@ except: pass wikipedia.output(u'%s\n\n' % returned_html) - wikipedia.output(u'%i' % response.code) + wikipedia.output(u'%i %s' % (response.status, response.reason))
if self.targetSite.mediawiki_message('uploadwarning') in returned_html: answer = wikipedia.inputChoice(u"You have recevied an upload warning message. Ignore?", ['Yes', 'No'], ['y', 'N'], 'N')
Modified: trunk/pywikipedia/userlib.py =================================================================== --- trunk/pywikipedia/userlib.py 2009-10-21 02:12:58 UTC (rev 7521) +++ trunk/pywikipedia/userlib.py 2009-10-21 02:31:09 UTC (rev 7522) @@ -554,7 +554,7 @@ address = self.site().unblock_address()
response, data = self.site().postForm(address, predata, sysop = True) - if response.code != 302: + if response.status != 302: if self.site().mediawiki_message('ipb_cant_unblock').replace('$1',blockID) in data: raise AlreadyUnblockedError raise UnblockError, data
Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2009-10-21 02:12:58 UTC (rev 7521) +++ trunk/pywikipedia/weblinkchecker.py 2009-10-21 02:31:09 UTC (rev 7522) @@ -394,7 +394,7 @@ return self.resolveRedirect(useHEAD = False) else: raise - if response.code >= 300 and response.code <= 399: + if response.status >= 300 and response.status <= 399: #print response.getheaders() redirTarget = response.getheader('Location') if redirTarget: @@ -493,10 +493,10 @@ # read the server's encoding, in case we need it later self.readEncodingFromResponse(response) # site down if the server status is between 400 and 499 - alive = response.code not in range(400, 500) - if response.code in self.HTTPignore: + alive = response.status not in range(400, 500) + if response.status in self.HTTPignore: alive = False - return alive, '%s' % response.code + return alive, '%s %s' % (response.status, response.reason)
class LinkCheckThread(threading.Thread): '''
Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-10-21 02:12:58 UTC (rev 7521) +++ trunk/pywikipedia/wikipedia.py 2009-10-21 02:31:09 UTC (rev 7522) @@ -123,7 +123,7 @@ __version__ = '$Id$'
import os, sys -import socket, urllib, urllib2 +import httplib, socket, urllib, urllib2 import traceback import time, threading, Queue import math @@ -1647,6 +1647,8 @@ response, data = query.GetData(params, self.site(), sysop=sysop, back_response = True) if query.IsString(data): raise KeyError + except httplib.BadStatusLine, line: + raise PageNotSaved('Bad status line: %s' % line.line) except ServerError: output(u''.join(traceback.format_exception(*sys.exc_info()))) retry_attempt += 1 @@ -1671,7 +1673,7 @@ # Check blocks self.site().checkBlocks(sysop = sysop) # A second text area means that an edit conflict has occured. - if response.code == 500: + if response.status == 500: output(u"Server error encountered; will retry in %i minute%s." % (retry_delay, retry_delay != 1 and "s" or "")) time.sleep(60 * retry_delay) @@ -1696,8 +1698,8 @@ #for debug only #------------------------ if verbose: - output("error occured, code:%s\ninfo:%s\nstatus:%s" % ( - data['error']['code'], data['error']['info'], response.code)) + output("error occured, code:%s\ninfo:%s\nstatus:%s\nresponse:%s" % ( + data['error']['code'], data['error']['info'], response.status, response.reason)) faked = params if 'text' in faked: del faked['text'] @@ -1706,7 +1708,7 @@ #------------------------ errorCode = data['error']['code'] #cannot handle longpageerror and PageNoSave yet - if errorCode == 'maxlag' or response.code == 503: + if errorCode == 'maxlag' or response.status == 503: # server lag; wait for the lag time and retry m = re.search('Waiting for (.+?): (.+?) seconds lagged', data['error']['info']) timelag = int(m.group(2)) @@ -1791,13 +1793,13 @@ # if the page update is successed, we need to return code 302 for cheat script who # using status code # - return 302, data['edit'] + return 302, response.reason, data['edit']
solve = self.site().solveCaptcha(data) if solve: return self._putPage(text, comment, watchArticle, minorEdit, newPage, token, newToken, sysop, captcha=solve)
- return response.code, data + return response.status, response.reason, data
def _putPageOld(self, text, comment=None, watchArticle=False, minorEdit=True, @@ -1884,7 +1886,7 @@ return None try: response, data = self.site().postForm(address, predata, sysop) - if response.code == 503: + if response.status == 503: if 'x-database-lag' in response.msg.keys(): # server lag; Mediawiki recommends waiting 5 seconds # and retrying @@ -1896,7 +1898,9 @@ wait = min(wait*2, 300) continue # Squid error 503 - raise ServerError(response.code) + raise ServerError(response.status) + except httplib.BadStatusLine, line: + raise PageNotSaved('Bad status line: %s' % line.line) except ServerError: output(u''.join(traceback.format_exception(*sys.exc_info()))) retry_attempt += 1 @@ -1988,7 +1992,7 @@ # to "Wikipedia has a problem", but I'm not sure. Maybe we could # just check for HTTP Status 500 (Internal Server Error)? if ("<title>Wikimedia Error</title>" in data or "has a problem</title>" in data) \ - or response.code == 500: + or response.status == 500: output(u"Server error encountered; will retry in %i minute%s." % (retry_delay, retry_delay != 1 and "s" or "")) time.sleep(60 * retry_delay) @@ -2049,7 +2053,7 @@ # Something went wrong, and we don't know what. Show the # HTML code that hopefully includes some error message. output(u"ERROR: Unexpected response from wiki server.") - output(u" %s" % response.code) + output(u" %s (%s) " % (response.status, response.reason)) output(data) # Unexpected responses should raise an error and not pass, # be it silently or loudly. This should raise an error @@ -2058,7 +2062,7 @@ # We are on the preview page, so the page was not saved raise PageNotSaved
- return response.code, data + return response.status, response.reason, data
def canBeEdited(self): """Return bool indicating whether this page can be edited. @@ -3281,7 +3285,7 @@ else: response, data = self.site().postForm(address, predata, sysop=True)
- if response.code == 302 and not data: + if response.status == 302 and not data: output(u'Changed protection level of page %s.' % self.aslink()) return True else: @@ -3289,7 +3293,7 @@ self.site().checkBlocks(sysop = True) output(u'Failed to change protection level of page %s:' % self.aslink()) - output(u"HTTP response code %s" % response.code) + output(u"HTTP response code %s" % response.status) output(data) return False
@@ -3687,7 +3691,7 @@ while True: try: data = self.getData() - except (socket.error, ServerError): + except (socket.error, httplib.BadStatusLine, ServerError): # Print the traceback of the caught exception output(u''.join(traceback.format_exception(*sys.exc_info()))) output(u'DBG> got network error in _GetAll.run. ' \ @@ -5134,6 +5138,14 @@ if not language[0].upper() + language[1:] in self.namespaces(): self._validlanguages.append(language)
+ #if persistent_http is None: + # persistent_http = config.persistent_http + #self.persistent_http = persistent_http and self.protocol() in ('http', 'https') + #if persistent_http: + # if self.protocol() == 'http': + # self.conn = httplib.HTTPConnection(self.hostname()) + # elif self.protocol() == 'https': + # self.conn = httplib.HTTPSConnection(self.hostname()) self.persistent_http = False
def _userIndex(self, sysop = False): @@ -5377,94 +5389,75 @@ """
# TODO: add the authenticate stuff here - if self.hostname() in config.authenticate.keys(): - uo = authenticateURLopener + + #if False: #self.persistent_http: + # conn = self.conn + #else: + if config.proxy['host']: + conn = httplib.HTTPConnection(config.proxy['host']) + proxyPutAddr = '%s://%s%s' % (self.protocol(), self.hostname(), address) + conn.putrequest('POST', proxyPutAddr) + if type(config.proxy['auth']) == tuple: + import base64 + authcode = base64.b64encode("%s:%s" % (config.proxy['auth'][0], config.proxy['auth'][1]) ) + conn.putheader('Proxy-Authorization', "Basic %s" % authcode ) + else: - if config.proxy['host'] and type(config.proxy['auth']) == tuple: - proxyHandle = {'http':'http://%s:%s@%s' % (config.proxy['auth'][0], config.proxy['auth'][1], config.proxy['host'] )} - elif config.proxy['host']: - proxyHandle = {'http':'http://%s' % config.proxy['host'] } - else: - proxyHandle = None + if self.protocol() == 'http': + conn = httplib.HTTPConnection(self.hostname()) + elif self.protocol() == 'https': + conn = httplib.HTTPSConnection(self.hostname())
- uo = MyURLopener(proxies = proxyHandle) - uo.addheader('Cookie', cookies) - if compress: - uo.addheader('Accept-encoding', 'gzip') + conn.putrequest('POST', address) + + # Encode all of this into a HTTP request + # otherwise, it will crash, as other protocols are not supported + if address[-1] == "?": address = address[:-1] - url = '%s://%s%s' % (self.protocol(), self.hostname(), address) + if self.hostname() in config.authenticate.keys(): + import base64 + authcode = base64.b64encode("%s:%s" % (config.authenticate[self.hostname()][0], config.authenticate[self.hostname()][1]) ) + conn.putheader("Authorization", "Basic %s" % authcode ) + + conn.putheader('Content-Length', str(len(data))) + conn.putheader('Content-type', contentType) + conn.putheader('User-agent', useragent) + if cookies: + conn.putheader('Cookie', cookies) + #if False: #self.persistent_http: + # conn.putheader('Connection', 'Keep-Alive') + if compress: + conn.putheader('Accept-encoding', 'gzip') + conn.endheaders() + conn.send(data)
- # Try to retrieve the page until it was successfully loaded (just in - # case the server is down or overloaded). - # Wait for retry_idle_time minutes (growing!) between retries. - retry_idle_time = 1 - while True: - try: - if self.hostname() in config.authenticate.keys(): - request = urllib2.Request(url, data) - request.add_header('User-agent', useragent) - opener = urllib2.build_opener() - f = opener.open(request) - else: - f = uo.open(url, data) + # Prepare the return values + # Note that this can raise network exceptions which are not + # caught here. + try: + response = conn.getresponse() + except httplib.BadStatusLine: + # Blub. + conn.close() + conn.connect() + return self.postData(address, data, contentType, sysop, compress, cookies)
- # read & info can raise socket.error - text = f.read() - headers = f.info() - break - except KeyboardInterrupt: - raise - except Exception, e: - if retry: - # We assume that the server is down. Wait some time, then try again. - output(u"%s" % e) - output(u"WARNING: Could not open '%s'. Maybe the server or" % url) - output(u"your connection is down. Retrying in %i minutes..." % retry_idle_time) - time.sleep(retry_idle_time * 60) - # Next time wait longer, but not longer than half an hour - retry_idle_time *= 2 - if retry_idle_time > 30: - retry_idle_time = 30 - else: - raise + data = response.read()
- contentType = headers.get('content-type', '') - contentEncoding = headers.get('content-encoding', '') + if compress and response.getheader('Content-Encoding') == 'gzip': + data = decompress_gzip(data)
- # Ensure that all sent data is received - if int(headers.get('content-length', '0')) != len(text) and 'content-length' in headers: - output(u'Warning! len(text) does not match content-length: %s != %s' % \ - (len(text), headers.get('content-length'))) - return self.postData(path, address, data, contentType, sysop, compress, cookie) + data = data.decode(self.encoding()) + response.close()
- if compress and contentEncoding == 'gzip': - text = decompress_gzip(text) + if True: #not self.persistent_http: + conn.close()
- R = re.compile('charset=([^'";]+)') - m = R.search(contentType) - if m: - charset = m.group(1) - else: - if verbose: - output(u"WARNING: No character set found.") - # UTF-8 as default - charset = 'utf-8' - # Check if this is the charset we expected - self.checkCharset(charset) - # Convert HTML to Unicode - try: - text = unicode(text, charset, errors = 'strict') - except UnicodeDecodeError, e: - print e - output(u'ERROR: Invalid characters found on %s://%s%s, replaced by \ufffd.' % (self.protocol(), self.hostname(), path)) - # We use error='replace' in case of bad encoding. - text = unicode(text, charset, errors = 'replace') - # If a wiki page, get user data - self._getUserDataOld(text, sysop = sysop) + self._getUserDataOld(data, sysop = sysop)
- return f, text + return response, data
def getUrl(self, path, retry = None, sysop = False, data = None, compress = True, no_hostname = False, cookie_only=False, back_response=False): @@ -5531,8 +5524,10 @@ if retry: # We assume that the server is down. Wait some time, then try again. output(u"%s" % e) - output(u"WARNING: Could not open '%s'. Maybe the server or" % url) - output("your connection is down. Retrying in %i minutes..." % retry_idle_time) + output(u"""\ +WARNING: Could not open '%s'. Maybe the server or +your connection is down. Retrying in %i minutes...""" + % (url, retry_idle_time)) time.sleep(retry_idle_time * 60) # Next time wait longer, but not longer than half an hour retry_idle_time *= 2 @@ -5550,6 +5545,9 @@ if int(headers.get('content-length', '0')) != len(text) and 'content-length' in headers: output(u'Warning! len(text) does not match content-length: %s != %s' % \ (len(text), headers.get('content-length'))) + #if False: #self.persistent_http + # self.conn.close() + # self.conn.connect() return self.getUrl(path, retry, sysop, data, compress, no_hostname, cookie_only, back_response)
if compress and contentEncoding == 'gzip':