http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10016

Revision: 10016 Author: xqt Date: 2012-03-12 14:12:02 +0000 (Mon, 12 Mar 2012) Log Message: ----------- strip trailing whitespace, keywords:Id

Modified Paths: -------------- trunk/threadedhttp/threadedhttp/__init__.py trunk/threadedhttp/threadedhttp/connectionpool.py trunk/threadedhttp/threadedhttp/cookiejar.py trunk/threadedhttp/threadedhttp/dummy.py trunk/threadedhttp/threadedhttp/http.py trunk/threadedhttp/threadedhttp/threadedhttp.py

Property Changed: ---------------- trunk/threadedhttp/threadedhttp/cookiejar.py trunk/threadedhttp/threadedhttp/dummy.py trunk/threadedhttp/threadedhttp/http.py trunk/threadedhttp/threadedhttp/threadedhttp.py

Modified: trunk/threadedhttp/threadedhttp/__init__.py =================================================================== --- trunk/threadedhttp/threadedhttp/__init__.py 2012-03-12 14:08:55 UTC (rev 10015) +++ trunk/threadedhttp/threadedhttp/__init__.py 2012-03-12 14:12:02 UTC (rev 10016) @@ -5,4 +5,4 @@ """

from http import Http -from threadedhttp import HttpRequest, HttpProcessor \ No newline at end of file +from threadedhttp import HttpRequest, HttpProcessor

Modified: trunk/threadedhttp/threadedhttp/connectionpool.py =================================================================== --- trunk/threadedhttp/threadedhttp/connectionpool.py 2012-03-12 14:08:55 UTC (rev 10015) +++ trunk/threadedhttp/threadedhttp/connectionpool.py 2012-03-12 14:12:02 UTC (rev 10016) @@ -31,7 +31,7 @@ self.connections = [None] * max_connections # fill known connections witn Nones self.clists = {} # 'id': (semaphore, lock, [connection1, connection2]) logging.log(1,'<%r>: initialized' % self) - + def __del__(self): """ Destructor to close all connections in the pool. Not completely thread-safe, as connections *could* return just @@ -46,7 +46,7 @@ del self.clists finally: self.lock.release() - + def pop_connection(self, identifier): """ Gets a connection from identifiers connection pool @param identifier The pool identifier @@ -93,13 +93,13 @@ self.lock.release() except Exception, e: logging.log(20,'<%r>: Exception raised level 2 | %r' % (self, e)) - clist.max.release() + clist.max.release() raise except Exception, e: logging.log(20,'<%r>: Exception raised level 1 | %r' % (self, e)) self.global_max.release() raise - + def push_connection(self, identifier, connection): """ Gets a connection from identifiers connection pool @param identifier The pool identifier @@ -126,20 +126,20 @@ self.connections = {} self.lock = threading.Lock() self.maxnum = maxnum - + def __del__(self): """ Destructor to close all connections in the pool """ self.lock.acquire() try: for connection in self.connections: connection.close() - + finally: self.lock.release() - + def __repr__(self): return self.connections.__repr__() - + def pop_connection(self, identifier): """ Gets a connection from identifiers connection pool @param identifier: The pool identifier @@ -153,7 +153,7 @@ return None finally: self.lock.release() - + def push_connection(self, identifier, connection): """ Adds a connection to identifiers connection pool @param identifier: The pool identifier @@ -163,7 +163,7 @@ try: if identifier not in self.connections: self.connections[identifier] = [] - + if len(self.connections[identifier]) == self.maxnum: logging.debug('closing %s connection %r' % (identifier, connection)) connection.close() @@ -171,4 +171,4 @@ else: self.connections[identifier].append(connection) finally: - self.lock.release() \ No newline at end of file + self.lock.release()

Modified: trunk/threadedhttp/threadedhttp/cookiejar.py =================================================================== --- trunk/threadedhttp/threadedhttp/cookiejar.py 2012-03-12 14:08:55 UTC (rev 10015) +++ trunk/threadedhttp/threadedhttp/cookiejar.py 2012-03-12 14:12:02 UTC (rev 10016) @@ -19,7 +19,7 @@ def __init__(self, *args, **kwargs): cookielib.CookieJar.__init__(self, *args, **kwargs) self.lock = threading.Lock() - + class LockableFileCookieJar(cookielib.FileCookieJar): """ CookieJar with integrated Lock object """ def __init__(self, *args, **kwargs):

Property changes on: trunk/threadedhttp/threadedhttp/cookiejar.py ___________________________________________________________________ Added: svn:keywords + Id

Modified: trunk/threadedhttp/threadedhttp/dummy.py =================================================================== --- trunk/threadedhttp/threadedhttp/dummy.py 2012-03-12 14:08:55 UTC (rev 10015) +++ trunk/threadedhttp/threadedhttp/dummy.py 2012-03-12 14:12:02 UTC (rev 10016) @@ -23,7 +23,7 @@ # copyright notice, this list of conditions and the following # disclaimer in the documentation and/or other materials provided # with the distribution. -# +# # THIS SOFTWARE IS PROVIDED BY METAWEB TECHNOLOGIES AND CONTRIBUTORS # ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS @@ -113,4 +113,4 @@ # as part of the expires= date format. so we have # to split carefully here - header.split(',') won't do it. HEADERVAL= re.compile(r'\s*(([^,]|(,\s*\d))+)') - return [h[0] for h in HEADERVAL.findall(self.response[k])] \ No newline at end of file + return [h[0] for h in HEADERVAL.findall(self.response[k])]

Property changes on: trunk/threadedhttp/threadedhttp/dummy.py ___________________________________________________________________ Added: svn:keywords + Id

Modified: trunk/threadedhttp/threadedhttp/http.py =================================================================== --- trunk/threadedhttp/threadedhttp/http.py 2012-03-12 14:08:55 UTC (rev 10015) +++ trunk/threadedhttp/threadedhttp/http.py 2012-03-12 14:12:02 UTC (rev 10016) @@ -4,7 +4,7 @@ # # (C) Merlijn van Deen, 2007 # -# Indicated parts (C) Joe Gregorio et al, 2006 +# Indicated parts (C) Joe Gregorio et al, 2006 # Distributed under the terms of the MIT license # __version__ = '$Id$' @@ -24,7 +24,7 @@

class Http(httplib2.Http): """ Subclass of httplib2.Http that uses a `LockableCookieJar` to store cookies. - Overrides httplib2s internal redirect support to prevent cookies + Overrides httplib2s internal redirect support to prevent cookies being eaten by the wrong sites. """ def __init__(self, *args, **kwargs): @@ -42,13 +42,13 @@ @param uri: The uri to retrieve @param method: (optional) The HTTP method to use. Default is 'GET' @param body: (optional) The request body. Default is no body. - @param headers: (optional) Additional headers to send. Defaults include + @param headers: (optional) Additional headers to send. Defaults include C{connection: keep-alive}, C{user-agent} and C{content-type}. @param max_redirects: (optional) The maximum number of redirects to use for this request. The class instances max_redirects is default @param connection_type: (optional) ? @returns: (response, content) tuple - """ + """ if max_redirects is None: max_redirects = self.max_redirects if headers is None: @@ -62,18 +62,18 @@ finally: self.cookiejar.lock.release() headers = req.headers - + # Wikimedia squids: add connection: keep-alive to request headers unless overridden headers['connection'] = headers.pop('connection', 'keep-alive') - + # determine connection pool key and fetch connection (scheme, authority, request_uri, defrag_uri) = httplib2.urlnorm(httplib2.iri2uri(uri)) conn_key = scheme+":"+authority - + connection = self.connection_pool.pop_connection(conn_key) if connection is not None: self.connections[conn_key] = connection - + # Redirect hack: we want to regulate redirects follow_redirects = self.follow_redirects #print 'follow_redirects: %r %r' % (self.follow_redirects, follow_redirects) @@ -84,30 +84,30 @@ #print 'follow_redirects: %r %r' % (self.follow_redirects, follow_redirects) self.follow_redirects = follow_redirects #print 'follow_redirects: %r %r' % (self.follow_redirects, follow_redirects) - - + + # return connection to pool self.connection_pool.push_connection(conn_key, self.connections[conn_key]) del self.connections[conn_key] - - # First write cookies + + # First write cookies self.cookiejar.lock.acquire() - try: + try: self.cookiejar.extract_cookies(DummyResponse(response), req) finally: self.cookiejar.lock.release() - + # Check for possible redirects redirectable_response = ((response.status == 303) or (response.status in [300, 301, 302, 307] and method in ["GET", "HEAD"])) if self.follow_redirects and (max_redirects > 0) and redirectable_response: (response, content) = self._follow_redirect(uri, method, body, headers, response, content, max_redirects) return (response, content) - + # The _follow_redirect function is based on the redirect handling in the # _request function of httplib2. The original function is (C) Joe Gregorio et al, 2006 # and licensed under the MIT license. Other contributers include - # Thomas Broyer (t.broyer@ltgt.net), James Antill, Xavier Verges Farrero, + # Thomas Broyer (t.broyer@ltgt.net), James Antill, Xavier Verges Farrero, # Jonathan Feinberg, Blair Zajac, Sam Ruby and Louis Nyffenegger (httplib2.__contributers__) def _follow_redirect(self, uri, method, body, headers, response, content, max_redirects): """ Internal function to follow a redirect recieved by L{request} """ @@ -131,15 +131,15 @@ if response.status == 301 and method in ["GET", "HEAD"]: response['-x-permanent-redirect-url'] = response['location'] if not response.has_key('content-location'): - response['content-location'] = absolute_uri + response['content-location'] = absolute_uri httplib2._updateCache(headers, response, content, self.cache, cachekey) - + headers.pop('if-none-match', None) headers.pop('if-modified-since', None) - + if response.has_key('location'): location = response['location'] redirect_method = ((response.status == 303) and (method not in ["GET", "HEAD"])) and "GET" or method return self.request(location, redirect_method, body=body, headers = headers, max_redirects = max_redirects - 1) else: - raise httplib2.RedirectLimit("Redirected more times than redirection_limit allows.", response, content) \ No newline at end of file + raise httplib2.RedirectLimit("Redirected more times than redirection_limit allows.", response, content)

Property changes on: trunk/threadedhttp/threadedhttp/http.py ___________________________________________________________________ Added: svn:keywords + Id

Modified: trunk/threadedhttp/threadedhttp/threadedhttp.py =================================================================== --- trunk/threadedhttp/threadedhttp/threadedhttp.py 2012-03-12 14:08:55 UTC (rev 10015) +++ trunk/threadedhttp/threadedhttp/threadedhttp.py 2012-03-12 14:12:02 UTC (rev 10016) @@ -21,7 +21,7 @@ >>> queue.put(request) >>> request.lock.acquire() >>> print request.data - + C{request.lock.acquire()} will block until the data is available. """ def __init__(self, *args, **kwargs): @@ -40,7 +40,7 @@ threading.Thread.__init__(self) self.queue = queue self.http = Http(cookiejar=cookiejar, connection_pool=connection_pool) - + def run(self): # The Queue item is expected to either an HttpRequest object # or None (to shut down the thread) @@ -54,4 +54,4 @@ item.data = self.http.request(*item.args, **item.kwargs) finally: if item.lock: - item.lock.release() \ No newline at end of file + item.lock.release()

Property changes on: trunk/threadedhttp/threadedhttp/threadedhttp.py ___________________________________________________________________ Added: svn:keywords + Id