SVN: [7576] trunk/pywikipedia - Pywikipedia-svn

31 Oct 2009

Revision: 7576
Author:   wikipedian
Date:     2009-10-31 00:35:16 +0000 (Sat, 31 Oct 2009)
Log Message:
-----------
used page generators to make the ignore list feature available to other 
scripts
Modified Paths:
--------------
    trunk/pywikipedia/pagegenerators.py
    trunk/pywikipedia/solve_disambiguation.py
Modified: trunk/pywikipedia/pagegenerators.py
===================================================================

--- trunk/pywikipedia/pagegenerators.py	2009-10-30 23:57:42 UTC (rev 7575)
+++ trunk/pywikipedia/pagegenerators.py	2009-10-31 00:35:16 UTC (rev 7576)
@@ -734,6 +734,32 @@
         if page.namespace() in namespaces:
             yield page
+def PageTitleFilterPageGenerator(generator, ignoreList):
+    """
+    Wraps around another generator. Yields only those pages are not
+    listed in the ignore list.
+
+    The ignoreList is a dictionary. Family names are mapped to
+    dictionaries in which language codes are mapped to lists of
+    page titles.
+    """
+
+    def isIgnored(page):
+        if not (page.site().family.name in ignoreList and page.site().lang in ignoreList[page.site().family.name]):
+            return False
+
+        for ig in ignoreList[page.site().family.name][page.site().lang]:
+            if re.match(ig, page.title()):
+                return True
+        return False
+
+    for page in generator:
+        if isIgnored(page):
+            if wikipedia.verbose:
+                wikipedia.output('Ignoring page %s' % page.title())
+        else:
+            yield page
+
 def RedirectFilterPageGenerator(generator):
     """
     Wraps around another generator. Yields only those pages that are not redirects.
Modified: trunk/pywikipedia/solve_disambiguation.py
===================================================================
--- trunk/pywikipedia/solve_disambiguation.py	2009-10-30 23:57:42 UTC (rev 7575)
+++ trunk/pywikipedia/solve_disambiguation.py	2009-10-31 00:35:16 UTC (rev 7576)
@@ -412,28 +412,14 @@
 class ReferringPageGeneratorWithIgnore:
     def __init__(self, disambPage, primary=False, minimum = 0):
         self.disambPage = disambPage
-        # if run with the -primary argument, enable the ignore manager
-        self.primaryIgnoreManager = PrimaryIgnoreManager(disambPage,
-                                                         enabled=primary)
         self.minimum = minimum
-        
+
     def __iter__(self):
-        # TODO: start yielding before all referring pages have been found
-        refs = [page for page in self.disambPage.getReferences(follow_redirects = False, withTemplateInclusion = False)]
-        pywikibot.output(u"Found %d references." % len(refs))
-        # Remove ignorables
-        if self.disambPage.site().family.name in ignore_title and self.disambPage.site().lang in ignore_title[self.disambPage.site().family.name]:
-            for ig in ignore_title[self.disambPage.site().family.name][self.disambPage.site().lang]:
-                for i in range(len(refs)-1, -1, -1):
-                    if re.match(ig, refs[i].title()):
-                        if pywikibot.verbose:
-                            pywikibot.output('Ignoring page %s'
-                                             % refs[i].title())
-                        del refs[i]
-        for i in range(len(refs)-1, -1, -1):
-            if self.primaryIgnoreManager.isIgnored(refs[i]):
-                #pywikibot.output('Ignoring page %s because it was skipped before' % refs[i].title())
-                del refs[i]
+        generator = pagegenerators.ReferringPageGenerator(self.disambPage, followRedirects  = False, withTemplateInclusion = False)
+        generator = pagegenerators.PageTitleFilterPageGenerator(generator, ignore_title)
+
+        refs = [page for page in generator]
+
         if len(refs) < self.minimum:
             pywikibot.output(u"Found only %d pages to work on; skipping." % len(refs))
             return

    