http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10074

Revision: 10074 Author: xqt Date: 2012-04-01 11:38:34 +0000 (Sun, 01 Apr 2012) Log Message: ----------- follow-up for r8574: use a generator for processing pages; use pagegenerators.PreloadingGenerator() for preloading pages decrease exhausting memory usage by cloning the pages before getting the content; fix featured category for cs-wiki fix featured template for cy-wiki main stuff to the main() procedure

Modified Paths: -------------- trunk/pywikipedia/featured.py

Modified: trunk/pywikipedia/featured.py =================================================================== --- trunk/pywikipedia/featured.py 2012-04-01 09:51:21 UTC (rev 10073) +++ trunk/pywikipedia/featured.py 2012-04-01 11:38:34 UTC (rev 10074) @@ -52,9 +52,11 @@ #

import sys, re, pickle, os.path +from copy import copy import wikipedia as pywikibot from pywikibot import i18n import catlib, config +from pagegenerators import PreloadingGenerator

def CAT(site, name, hide): name = site.namespace(14) + ':' + name @@ -141,8 +143,8 @@ 'bs': (CAT, u"Odabrani članci"), 'ca': (CAT, u"Llista d'articles de qualitat"), 'ceb':(CAT, u"Mga napiling artikulo"), - 'cs': (CAT, u"Nejlepší články"), - 'cy': (CAT, u"Erthyglau dethol"), + 'cs': (CAT, u"Wikipedie:Nejlepší články"), + 'cy': (BACK,u"Erthygl ddethol"), 'da': (CAT, u"Fremragende artikler"), 'de': (CAT, u"Wikipedia:Exzellent"), #'dsb':(CAT, u"Ekscelentny"), @@ -301,14 +303,8 @@ 'zh': (CAT, u"Wikipedia_former_featured_articles"), }

-# globals -interactive=0 -nocache=0 -afterpage=u"!" -cache={} - def featuredArticles(site, pType): - arts=[] + articles=[] if pType == 'good': info = good_name elif pType == 'former': @@ -323,7 +319,7 @@ pywikibot.output( u'Error: language %s doesn't has %s category source.' % (site.lang, pType)) - return arts + return name = info[site.lang][1] # hide #-sorted items on en-wiki try: @@ -333,14 +329,16 @@ raw = method(site, name, hide) for p in raw: if p.namespace() == 0: # Article - arts.append(p) + articles.append(p) # Article talk (like in English) elif p.namespace() == 1 and site.lang <> 'el': - arts.append(pywikibot.Page(p.site(), p.title(withNamespace=False))) + articles.append(pywikibot.Page(p.site(), + p.title(withNamespace=False))) pywikibot.output( '\03{lightred}** wikipedia:%s has %i %s articles\03{default}' - % (site.lang, len(arts), pType)) - return arts + % (site.lang, len(articles), pType)) + for p in articles: + yield copy(p)

def findTranslated(page, oursite=None, quiet=False): if not oursite: @@ -441,10 +439,11 @@ fromsite.lang), re.IGNORECASE) re_this_iw=re.compile(ur"[[%s:[^]]+]]" % fromsite.lang)

- arts = featuredArticles(fromsite, pType) + gen = featuredArticles(fromsite, pType) + gen = PreloadingGenerator(gen)

pairs=[] - for a in arts: + for a in gen: if a.title() < afterpage: continue if u"/" in a.title() and a.namespace() != 0: @@ -540,7 +539,13 @@ except pywikibot.PageNotSaved, e: pywikibot.output(u"Page not saved")

-if __name__=="__main__": +def main(*args): + global nocache, interactive, afterpage, cache + nocache = 0 + interactive = 0 + afterpage = u"!" + cache = {} + template_on_top = True featuredcount = False fromlang=[] @@ -632,7 +637,10 @@ for ll in fromlang: fromsite = pywikibot.getSite(ll) if featuredcount: - featuredArticles(fromsite, processType) + try: + featuredArticles(fromsite, processType).next() + except StopIteration: + continue elif not hasTemplate: pywikibot.output( u'\nNOTE: %s arcticles are not implemented at %s-wiki.' @@ -645,6 +653,11 @@ except KeyboardInterrupt: pywikibot.output('\nQuitting program...') finally: - pywikibot.stopme() if not nocache: pickle.dump(cache,file(filename,"wb")) + +if __name__ == "__main__": + try: + main() + finally: + pywikibot.stopme()