Revision: 7433
Author: alexsh
Date: 2009-10-12 08:35:02 +0000 (Mon, 12 Oct 2009)
Log Message:
-----------
Site().postData() and Site().getUrl(): add proxy support (included basic authorization)
Modified Paths:
--------------
trunk/pywikipedia/config.py
trunk/pywikipedia/wikipedia.py
Modified: trunk/pywikipedia/config.py
===================================================================
--- trunk/pywikipedia/config.py 2009-10-12 05:26:55 UTC (rev 7432)
+++ trunk/pywikipedia/config.py 2009-10-12 08:35:02 UTC (rev 7433)
@@ -343,6 +343,8 @@
}
# for all connection proxy handle
+# to use it, proxy['host'] have to support HTTP and include port number (e.g. localhost:8080)
+# if proxy server neen authentication, set ('ID', 'PASSWORD') to proxy['auth'].
proxy = {
'host': None,
'auth': None,
Modified: trunk/pywikipedia/wikipedia.py
===================================================================
--- trunk/pywikipedia/wikipedia.py 2009-10-12 05:26:55 UTC (rev 7432)
+++ trunk/pywikipedia/wikipedia.py 2009-10-12 08:35:02 UTC (rev 7433)
@@ -5055,14 +5055,32 @@
#if False: #self.persistent_http:
# conn = self.conn
#else:
- # Encode all of this into a HTTP request
if self.protocol() == 'http':
- conn = httplib.HTTPConnection(self.hostname())
+ if config.proxy['host']:
+ conn = httplib.HTTPConnection(config.proxy['host'])
+ else:
+ conn = httplib.HTTPConnection(self.hostname())
elif self.protocol() == 'https':
conn = httplib.HTTPSConnection(self.hostname())
+ # Encode all of this into a HTTP request
# otherwise, it will crash, as other protocols are not supported
- conn.putrequest('POST', address)
+ if address[-1] == "?":
+ address = address[:-1]
+ if config.proxy['host']:
+ proxyPutAddr = '%s://%s%s' % (self.protocol(), self.hostname(), address)
+ conn.putrequest('POST', proxyPutAddr)
+ if type(config.proxy['auth']) == tuple:
+ import base64
+ authcode = base64.b64encode("%s:%s" % (config.proxy['auth'][0], config.proxy['auth'][1]) )
+ conn.putheader('Proxy-Authorization', "Basic %s" % authcode )
+ else:
+ conn.putrequest('POST', address)
+ if self.hostname() in config.authenticate.keys():
+ import base64
+ authcode = base64.b64encode("%s:%s" % (config.authenticate[self.hostname()][0], config.authenticate[self.hostname()][1]) )
+ conn.putheader("Authorization", "Basic %s" % authcode )
+
conn.putheader('Content-Length', str(len(data)))
conn.putheader('Content-type', contentType)
conn.putheader('User-agent', useragent)
@@ -5148,7 +5166,14 @@
if self.hostname() in config.authenticate.keys():
uo = authenticateURLopener
else:
- uo = MyURLopener()
+ if config.proxy['host'] and type(config.proxy['auth']) == tuple:
+ proxyHandle = {'http':'http://%s:%s@%s' % (config.proxy['auth'][0], config.proxy['auth'][1], config.proxy['host'] )}
+ elif config.proxy['host']:
+ proxyHandle = {'http':'http://%s' % config.proxy['host'] }
+ else:
+ proxyHandle = None
+
+ uo = MyURLopener(proxies = proxyHandle)
if self.cookies(sysop = sysop):
uo.addheader('Cookie', self.cookies(sysop = sysop))
if compress:
@@ -5163,8 +5188,7 @@
# case the server is down or overloaded).
# Wait for retry_idle_time minutes (growing!) between retries.
retry_idle_time = 1
- retrieved = False
- while not retrieved:
+ while True:
try:
if self.hostname() in config.authenticate.keys():
request = urllib2.Request(url, data)
@@ -5178,7 +5202,7 @@
text = f.read()
headers = f.info()
- retrieved = True
+ break
except KeyboardInterrupt:
raise
except Exception, e:
@@ -5188,8 +5212,7 @@
output(u"""\
WARNING: Could not open '%s'. Maybe the server or
your connection is down. Retrying in %i minutes..."""
- % (url,
- retry_idle_time))
+ % (url, retry_idle_tie))
time.sleep(retry_idle_time * 60)
# Next time wait longer, but not longer than half an hour
retry_idle_time *= 2
@@ -7787,7 +7810,16 @@
for site in config.authenticate:
passman.add_password(None, site, config.authenticate[site][0], config.authenticate[site][1])
authhandler = urllib2.HTTPBasicAuthHandler(passman)
- authenticateURLopener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),authhandler)
+ if config.proxy['host']:
+ proxyHandle = urllib2.ProxyHandler({'http':'http://%s' % config.proxy['host'] })
+ if config.proxy['auth']:
+ proxyAuth = urllib2.HTTPPasswordMgr()
+ proxyAuth.add_password(None, config.proxy['host'], config.proxy['auth'][0], config.proxy['auth'][1])
+ proxyAuthHandle = urllib2.ProxyBasicAuthHandler(proxyAuth)
+ else:
+ proxyHandle = None
+ proxyAuthHandle = None
+ authenticateURLopener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj), proxyHandle, proxyAuthHandle, authhandler)
urllib2.install_opener(authenticateURLopener)
if __name__ == '__main__':