Revision: 8574 Author: amir Date: 2010-09-19 12:32:27 +0000 (Sun, 19 Sep 2010)
Log Message: ----------- -query feature for use at small amount of RAM(e.g. toolserver)
Modified Paths: -------------- trunk/pywikipedia/featured.py
Modified: trunk/pywikipedia/featured.py =================================================================== --- trunk/pywikipedia/featured.py 2010-09-19 12:30:58 UTC (rev 8573) +++ trunk/pywikipedia/featured.py 2010-09-19 12:32:27 UTC (rev 8574) @@ -37,6 +37,10 @@
-dry for debug purposes. No changes will be made.
+-query:# a int. that determain number of pages will be checked each while + (use for computers with a small amount of RAM e.g. toolserver users) + default is 500 + usage: featured.py [-interactive] [-nocache] [-top] [-after:zzzz] [-fromlang:xx,yy--zz|-fromall]
""" @@ -515,9 +519,120 @@ except KeyError: templates = template['_default'] return templates +def featuredbot(arts,cc,tosite,template_on_top,pType, quiet,dry): + templatelist = getTemplateList(tosite.lang, pType) + findtemplate = '(' + '|'.join(templatelist) + ')' + re_Link_FA=re.compile(ur"{{%s|%s}}" + % (findtemplate.replace(u' ', u'[ _]'), + fromsite.lang), re.IGNORECASE) + re_this_iw=re.compile(ur"[[%s:[^]]+]]" % fromsite.lang) + pairs=[] + for a in arts: + if a.title()<afterpage: + continue + if u"/" in a.title() and a.namespace() != 0: + pywikibot.output(u"%s is a subpage" % a.title()) + continue + if a.title() in cc: + pywikibot.output(u"(cached) %s -> %s"%(a.title(), cc[a.title()])) + continue + if a.isRedirectPage(): + a=a.getRedirectTarget() + try: + if not a.exists(): + pywikibot.output(u"source page doesn't exist: %s" % a.title()) + continue + atrans = findTranslated(a, tosite, quiet) + if pType!='former': + if atrans: + text=atrans.get() + m=re_Link_FA.search(text) + if m: + pywikibot.output(u"(already done)") + else: + # insert just before interwiki + if (not interactive or + pywikibot.input( + u'Connecting %s -> %s. Proceed? [Y/N]' + % (a.title(), atrans.title())) in ['Y','y'] + ): + m=re_this_iw.search(text) + if not m: + pywikibot.output( + u"no interwiki record, very strange") + continue + site = pywikibot.getSite() + if pType == 'good': + comment = pywikibot.setAction( + pywikibot.translate(site, msg_good) + % (fromsite.lang, a.title())) + elif pType == 'list': + comment = pywikibot.setAction( + pywikibot.translate(site, msg_lists) + % (fromsite.lang, a.title())) + else: + comment = pywikibot.setAction( + pywikibot.translate(site, msg) + % (fromsite.lang, a.title())) + ### Moving {{Link FA|xx}} to top of interwikis ### + if template_on_top == True: + # Getting the interwiki + iw = pywikibot.getLanguageLinks(text, site) + # Removing the interwiki + text = pywikibot.removeLanguageLinks(text, site) + text += u"\r\n{{%s|%s}}\r\n" % (templatelist[0], + fromsite.lang) + # Adding the interwiki + text = pywikibot.replaceLanguageLinks(text, + iw, site)
+ ### Placing {{Link FA|xx}} right next to corresponding interwiki ### + else: + text=(text[:m.end()] + + (u" {{%s|%s}}" % (templatelist[0], + fromsite.lang)) + + text[m.end():]) + if not dry: + try: + atrans.put(text, comment) + except pywikibot.LockedPage: + pywikibot.output(u'Page %s is locked!' + % atrans.title()) + cc[a.title()]=atrans.title() + else: + if atrans: + text=atrans.get() + m=re_Link_FA.search(text) + if m: + # insert just before interwiki + if (not interactive or + pywikibot.input( + u'Connecting %s -> %s. Proceed? [Y/N]' + % (a.title(), atrans.title())) in ['Y','y'] + ): + m=re_this_iw.search(text) + if not m: + pywikibot.output( + u"no interwiki record, very strange") + continue + site = pywikibot.getSite() + comment = pywikibot.setAction( + pywikibot.translate(site, msg_former) + % (fromsite.lang, a.title())) + text=re.sub(re_Link_FA,'',text) + if not dry: + try: + atrans.put(text, comment) + except pywikibot.LockedPage: + pywikibot.output(u'Page %s is locked!' + % atrans.title()) + else: + pywikibot.output(u"(already done)") + cc[a.title()]=atrans.title() + except pywikibot.PageNotSaved, e: + pywikibot.output(u"Page not saved") def featuredWithInterwiki(fromsite, tosite, template_on_top, pType, quiet, - dry=False): + dry=False,query=500): if not fromsite.lang in cache: cache[fromsite.lang]={} if not tosite.lang in cache[fromsite.lang]: @@ -526,121 +641,16 @@ if nocache: cc={}
- templatelist = getTemplateList(tosite.lang, pType) - findtemplate = '(' + '|'.join(templatelist) + ')' - re_Link_FA=re.compile(ur"{{%s|%s}}" - % (findtemplate.replace(u' ', u'[ _]'), - fromsite.lang), re.IGNORECASE) - re_this_iw=re.compile(ur"[[%s:[^]]+]]" % fromsite.lang) - arts=featuredArticles(fromsite, pType) + top=0 + if len(arts)>query: + while top<len(arts): + bottom=top + top=top+query + featuredbot(arts[bottom:top],cc,tosite,template_on_top, pType, quiet,dry) + else: + featuredbot(arts,cc,tosite,template_on_top,pType,quiet,dry)
- pairs=[] - for a in arts: - if a.title()<afterpage: - continue - if u"/" in a.title() and a.namespace() != 0: - pywikibot.output(u"%s is a subpage" % a.title()) - continue - if a.title() in cc: - pywikibot.output(u"(cached) %s -> %s"%(a.title(), cc[a.title()])) - continue - if a.isRedirectPage(): - a=a.getRedirectTarget() - try: - if not a.exists(): - pywikibot.output(u"source page doesn't exist: %s" % a.title()) - continue - atrans = findTranslated(a, tosite, quiet) - if pType!='former': - if atrans: - text=atrans.get() - m=re_Link_FA.search(text) - if m: - pywikibot.output(u"(already done)") - else: - # insert just before interwiki - if (not interactive or - pywikibot.input( - u'Connecting %s -> %s. Proceed? [Y/N]' - % (a.title(), atrans.title())) in ['Y','y'] - ): - m=re_this_iw.search(text) - if not m: - pywikibot.output( - u"no interwiki record, very strange") - continue - site = pywikibot.getSite() - if pType == 'good': - comment = pywikibot.setAction( - pywikibot.translate(site, msg_good) - % (fromsite.lang, a.title())) - elif pType == 'list': - comment = pywikibot.setAction( - pywikibot.translate(site, msg_lists) - % (fromsite.lang, a.title())) - else: - comment = pywikibot.setAction( - pywikibot.translate(site, msg) - % (fromsite.lang, a.title())) - ### Moving {{Link FA|xx}} to top of interwikis ### - if template_on_top == True: - # Getting the interwiki - iw = pywikibot.getLanguageLinks(text, site) - # Removing the interwiki - text = pywikibot.removeLanguageLinks(text, site) - text += u"\r\n{{%s|%s}}\r\n" % (templatelist[0], - fromsite.lang) - # Adding the interwiki - text = pywikibot.replaceLanguageLinks(text, - iw, site) - - ### Placing {{Link FA|xx}} right next to corresponding interwiki ### - else: - text=(text[:m.end()] - + (u" {{%s|%s}}" % (templatelist[0], - fromsite.lang)) - + text[m.end():]) - if not dry: - try: - atrans.put(text, comment) - except pywikibot.LockedPage: - pywikibot.output(u'Page %s is locked!' - % atrans.title()) - cc[a.title()]=atrans.title() - else: - if atrans: - text=atrans.get() - m=re_Link_FA.search(text) - if m: - # insert just before interwiki - if (not interactive or - pywikibot.input( - u'Connecting %s -> %s. Proceed? [Y/N]' - % (a.title(), atrans.title())) in ['Y','y'] - ): - m=re_this_iw.search(text) - if not m: - pywikibot.output( - u"no interwiki record, very strange") - continue - site = pywikibot.getSite() - comment = pywikibot.setAction( - pywikibot.translate(site, msg_former) - % (fromsite.lang, a.title())) - text=re.sub(re_Link_FA,'',text) - if not dry: - try: - atrans.put(text, comment) - except pywikibot.LockedPage: - pywikibot.output(u'Page %s is locked!' - % atrans.title()) - else: - pywikibot.output(u"(already done)") - cc[a.title()]=atrans.title() - except pywikibot.PageNotSaved, e: - pywikibot.output(u"Page not saved") - if __name__=="__main__": template_on_top = True featuredcount = False @@ -650,6 +660,7 @@ part = False quiet = False dry = False + query=500 for arg in pywikibot.handleArgs(): if arg == '-interactive': interactive=1 @@ -658,6 +669,11 @@ elif arg.startswith('-fromlang:'): fromlang=arg[10:].split(",") part = True + elif arg.startswith('-query:'): + try: + query=int(arg[7:]) + except: + query=500 elif arg == '-fromall': doAll = True elif arg.startswith('-after:'): @@ -743,7 +759,7 @@ break elif fromsite != pywikibot.getSite(): featuredWithInterwiki(fromsite, pywikibot.getSite(), - template_on_top, processType, quiet, dry) + template_on_top, processType, quiet, dry,query) except KeyboardInterrupt: pywikibot.output('\nQuitting program...') finally: