Revision: 6038
Author: nicdumz
Date: 2008-10-28 10:23:45 +0000 (Tue, 28 Oct 2008)
Log Message:
-----------
Fix for [2193942 ] reading category: memory leak and slow down :
Let's not assume users want to crawl several times a category, and make the default
behavior NON CACHING. Why would a user iterate several times on a category content anyway
?
Modified Paths:
--------------
trunk/pywikipedia/catlib.py
Modified: trunk/pywikipedia/catlib.py
===================================================================
--- trunk/pywikipedia/catlib.py 2008-10-27 20:51:31 UTC (rev 6037)
+++ trunk/pywikipedia/catlib.py 2008-10-28 10:23:45 UTC (rev 6038)
@@ -93,7 +93,7 @@
else:
return '[[%s]]' % titleWithSortKey
- def _getContents(self, recurse=False, purge=False, startFrom=None, cache=None):
+ def _getAndCacheContents(self, recurse=False, purge=False, startFrom=None,
cache=None):
"""
Cache results of _parseCategory for a second call.
@@ -129,7 +129,7 @@
# contents of subcategory are cached by calling
# this method recursively; therefore, do not cache
# them again
- for item in subcat._getContents(newrecurse, purge, cache=cache):
+ for item in subcat._getAndCacheContents(newrecurse, purge,
cache=cache):
yield item
else:
for tag, page in self._parseCategory(purge, startFrom):
@@ -147,11 +147,22 @@
# contents of subcategory are cached by calling
# this method recursively; therefore, do not cache
# them again
- for item in page._getContents(newrecurse, purge,
cache=cache):
+ for item in page._getAndCacheContents(newrecurse, purge,
cache=cache):
yield item
if not startFrom:
self.completelyCached = True
+ def _getContentsNaive(self, recurse=False, startFrom=None):
+ """
+ Simple category content yielder. Naive, do not attempts to
+ cache anything
+ """
+ for tag, page in self._parseCategory(startFrom=startFrom):
+ yield tag, page
+ if tag == SUBCATEGORY and recurse:
+ for item in page._getContentsNaive(recurse=True):
+ yield item
+
def _parseCategory(self, purge=False, startFrom=None):
"""
Yields all articles and subcategories that are in this category.
@@ -259,7 +270,7 @@
else:
break
- def subcategories(self, recurse=False, startFrom=None):
+ def subcategories(self, recurse=False, startFrom=None, cacheResults=False):
"""
Yields all subcategories of the current category.
@@ -269,9 +280,18 @@
equivalent to recurse = False, recurse = 1 gives first-level
subcategories of subcategories but no deeper, etcetera).
+ cacheResults - cache the category contents: useful if you need to
+ do several passes on the category members list. The simple cache
+ system is *not* meant to be memory or cpu efficient for large
+ categories
+
Results a sorted (as sorted by MediaWiki), but need not be unique.
"""
- for tag, subcat in self._getContents(recurse, startFrom=startFrom):
+ if cacheResults:
+ gen = self._getAndCacheContents
+ else:
+ gen = self._getContentsNaive
+ for tag, subcat in gen(recurse=recurse, startFrom=startFrom):
if tag == SUBCATEGORY:
yield subcat
@@ -289,7 +309,7 @@
subcats.append(cat)
return unique(subcats)
- def articles(self, recurse=False, startFrom=None):
+ def articles(self, recurse=False, startFrom=None, cacheResults=False):
"""
Yields all articles of the current category.
@@ -297,10 +317,19 @@
Recurse can be a number to restrict the depth at which subcategories
are included.
+ cacheResults - cache the category contents: useful if you need to
+ do several passes on the category members list. The simple cache
+ system is *not* meant to be memory or cpu efficient for large
+ categories
+
Results are unsorted (except as sorted by MediaWiki), and need not
be unique.
"""
- for tag, page in self._getContents(recurse, startFrom=startFrom):
+ if cacheResults:
+ gen = self._getAndCacheContents
+ else:
+ gen = self._getContentsNaive
+ for tag, page in gen(recurse=recurse, startFrom=startFrom):
if tag == ARTICLE:
yield page
@@ -342,7 +371,7 @@
def isEmpty(self):
# TODO: rename; naming conflict with Page.isEmpty
- for tag, title in self._getContents(purge = True):
+ for tag, title in self._parseCategory():
return False
return True