http://www.mediawiki.org/wiki/Special:Code/pywikipedia/10074
Revision: 10074
Author: xqt
Date: 2012-04-01 11:38:34 +0000 (Sun, 01 Apr 2012)
Log Message:
-----------
follow-up for r8574:
use a generator for processing pages;
use pagegenerators.PreloadingGenerator() for preloading pages
decrease exhausting memory usage by cloning the pages before getting the content;
fix featured category for cs-wiki
fix featured template for cy-wiki
main stuff to the main() procedure
Modified Paths:
--------------
trunk/pywikipedia/featured.py
Modified: trunk/pywikipedia/featured.py
===================================================================
--- trunk/pywikipedia/featured.py 2012-04-01 09:51:21 UTC (rev 10073)
+++ trunk/pywikipedia/featured.py 2012-04-01 11:38:34 UTC (rev 10074)
@@ -52,9 +52,11 @@
#
import sys, re, pickle, os.path
+from copy import copy
import wikipedia as pywikibot
from pywikibot import i18n
import catlib, config
+from pagegenerators import PreloadingGenerator
def CAT(site, name, hide):
name = site.namespace(14) + ':' + name
@@ -141,8 +143,8 @@
'bs': (CAT, u"Odabrani članci"),
'ca': (CAT, u"Llista d'articles de qualitat"),
'ceb':(CAT, u"Mga napiling artikulo"),
- 'cs': (CAT, u"Nejlepší články"),
- 'cy': (CAT, u"Erthyglau dethol"),
+ 'cs': (CAT, u"Wikipedie:Nejlepší články"),
+ 'cy': (BACK,u"Erthygl ddethol"),
'da': (CAT, u"Fremragende artikler"),
'de': (CAT, u"Wikipedia:Exzellent"),
#'dsb':(CAT, u"Ekscelentny"),
@@ -301,14 +303,8 @@
'zh': (CAT, u"Wikipedia_former_featured_articles"),
}
-# globals
-interactive=0
-nocache=0
-afterpage=u"!"
-cache={}
-
def featuredArticles(site, pType):
- arts=[]
+ articles=[]
if pType == 'good':
info = good_name
elif pType == 'former':
@@ -323,7 +319,7 @@
pywikibot.output(
u'Error: language %s doesn\'t has %s category source.'
% (site.lang, pType))
- return arts
+ return
name = info[site.lang][1]
# hide #-sorted items on en-wiki
try:
@@ -333,14 +329,16 @@
raw = method(site, name, hide)
for p in raw:
if p.namespace() == 0: # Article
- arts.append(p)
+ articles.append(p)
# Article talk (like in English)
elif p.namespace() == 1 and site.lang <> 'el':
- arts.append(pywikibot.Page(p.site(), p.title(withNamespace=False)))
+ articles.append(pywikibot.Page(p.site(),
+ p.title(withNamespace=False)))
pywikibot.output(
'\03{lightred}** wikipedia:%s has %i %s articles\03{default}'
- % (site.lang, len(arts), pType))
- return arts
+ % (site.lang, len(articles), pType))
+ for p in articles:
+ yield copy(p)
def findTranslated(page, oursite=None, quiet=False):
if not oursite:
@@ -441,10 +439,11 @@
fromsite.lang), re.IGNORECASE)
re_this_iw=re.compile(ur"\[\[%s:[^]]+\]\]" % fromsite.lang)
- arts = featuredArticles(fromsite, pType)
+ gen = featuredArticles(fromsite, pType)
+ gen = PreloadingGenerator(gen)
pairs=[]
- for a in arts:
+ for a in gen:
if a.title() < afterpage:
continue
if u"/" in a.title() and a.namespace() != 0:
@@ -540,7 +539,13 @@
except pywikibot.PageNotSaved, e:
pywikibot.output(u"Page not saved")
-if __name__=="__main__":
+def main(*args):
+ global nocache, interactive, afterpage, cache
+ nocache = 0
+ interactive = 0
+ afterpage = u"!"
+ cache = {}
+
template_on_top = True
featuredcount = False
fromlang=[]
@@ -632,7 +637,10 @@
for ll in fromlang:
fromsite = pywikibot.getSite(ll)
if featuredcount:
- featuredArticles(fromsite, processType)
+ try:
+ featuredArticles(fromsite, processType).next()
+ except StopIteration:
+ continue
elif not hasTemplate:
pywikibot.output(
u'\nNOTE: %s arcticles are not implemented at %s-wiki.'
@@ -645,6 +653,11 @@
except KeyboardInterrupt:
pywikibot.output('\nQuitting program...')
finally:
- pywikibot.stopme()
if not nocache:
pickle.dump(cache,file(filename,"wb"))
+
+if __name__ == "__main__":
+ try:
+ main()
+ finally:
+ pywikibot.stopme()