Pywikipedia-svn July 2013

pywikipedia-svn@lists.wikimedia.org

5 participants
77 discussions

SVN: [11721] branches/rewrite/scripts/featured.py
by xqt＠svn.wikimedia.org 08 Jul '13

08 Jul '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11721 Revision: 11721 Author: xqt Date: 2013-07-07 16:44:15 +0000 (Sun, 07 Jul 2013) Log Message: ----------- pep9 changes, adopted for rewrite branch Modified Paths: -------------- branches/rewrite/scripts/featured.py Modified: branches/rewrite/scripts/featured.py =================================================================== --- branches/rewrite/scripts/featured.py 2013-07-07 15:48:09 UTC (rev 11720) +++ branches/rewrite/scripts/featured.py 2013-07-07 16:44:15 UTC (rev 11721) @@ -43,30 +43,35 @@ # # (C) Maxim Razin, 2005 # (C) Leonardo Gregianin, 2005-2008 -# (C) xqt, 2009-2012 +# (C) xqt, 2009-2013 # (C) Pywikipedia bot team, 2005-2012 # # Distributed under the terms of the MIT license. # -import sys, re, pickle, os.path +import os.path +import pickle +import re +import sys from copy import copy -import wikipedia as pywikibot +import pywikibot from pywikibot import i18n -import catlib, config -from pagegenerators import PreloadingGenerator +from pywikibot import config +from pywikibot.pagegenerators import PreloadingGenerator + def CAT(site, name, hide): name = site.namespace(14) + ':' + name - cat=catlib.Category(site, name) + cat = pywikibot.Category(site, name) for article in cat.articles(endsort=hide): yield article if hide: - for article in cat.articles(startFrom=unichr(ord(hide)+1)): + for article in cat.articles(startFrom=unichr(ord(hide) + 1)): yield article + def BACK(site, name, hide): - p=pywikibot.Page(site, name, defaultNamespace=10) + p = pywikibot.Page(site, name, defaultNamespace=10) return [page for page in p.getReferences(follow_redirects=False, onlyTemplateInclusion=True)] @@ -77,7 +82,7 @@ 'als': ['LinkFA'], 'an': ['Destacato', 'Destacau'], 'ar': [u'وصلة مقالة مختارة'], - 'ast':['Enllaz AD'], + 'ast': ['Enllaz AD'], 'az': ['Link FM'], 'br': ['Liamm PuB', 'Lien AdQ'], 'ca': [u'Enllaç AD', 'Destacat'], @@ -87,7 +92,7 @@ 'es': ['Destacado'], 'eu': ['NA lotura'], 'fr': ['Lien AdQ'], - 'fur':['Leam VdC'], + 'fur': ['Leam VdC'], 'ga': ['Nasc AR'], 'hi': ['Link FA', 'Lien AdQ'], 'is': [u'Tengill ÚG'], @@ -114,7 +119,7 @@ 'nn': ['Link AA'], 'no': ['Link AA'], 'pt': ['Bom interwiki'], - #'tr': ['Link GA', 'Link KM'], +## 'tr': ['Link GA', 'Link KM'], 'vi': [u'Liên kết bài chất lượng tốt'], 'wo': ['Lien BA'], } @@ -125,77 +130,77 @@ } featured_name = { - 'af': (BACK,u"Voorbladster"), - 'als':(CAT, u"Wikipedia:Bsunders glungener Artikel"), + 'af': (BACK, u"Voorbladster"), + 'als': (CAT, u"Wikipedia:Bsunders glungener Artikel"), 'am': (CAT, u"Wikipedia:Featured article"), 'an': (CAT, u"Articlos destacatos"), - 'ang':(CAT, u"Fulgōd ȝeƿritu"), + 'ang': (CAT, u"Fulgōd ȝeƿritu"), 'ar': (CAT, u"مقالات مختارة"), - 'ast':(CAT, u"Uiquipedia:Artículos destacaos"), - 'az': (BACK,u"Seçilmiş məqalə"), - 'bar':(CAT, u"Berig"), + 'ast': (CAT, u"Uiquipedia:Artículos destacaos"), + 'az': (BACK, u"Seçilmiş məqalə"), + 'bar': (CAT, u"Berig"), 'bat-smg': (CAT, u"Vikipedėjės pavīzdėnē straipsnē"), - 'be-x-old':(CAT, u"Вікіпэдыя:Абраныя артыкулы"), + 'be-x-old': (CAT, u"Вікіпэдыя:Абраныя артыкулы"), 'bg': (CAT, u"Избрани статии"), - 'bn': (BACK,u"নির্বাচিত নিবন্ধ"), + 'bn': (BACK, u"নির্বাচিত নিবন্ধ"), 'br': (CAT, u"Pennadoù eus an dibab"), 'bs': (CAT, u"Odabrani članci"), 'ca': (CAT, u"Llista d'articles de qualitat"), - 'ceb':(CAT, u"Mga napiling artikulo"), + 'ceb': (CAT, u"Mga napiling artikulo"), 'cs': (CAT, u"Wikipedie:Nejlepší články"), - 'cy': (BACK,u"Erthygl ddethol"), + 'cy': (BACK, u"Erthygl ddethol"), 'da': (CAT, u"Fremragende artikler"), 'de': (CAT, u"Wikipedia:Exzellent"), 'dv': (BACK, u"Featured article"), - 'el': (BACK,u"Αξιόλογο άρθρο"), + 'el': (BACK, u"Αξιόλογο άρθρο"), 'en': (CAT, u"Featured articles"), 'eo': (CAT, u"Elstaraj artikoloj"), 'es': (BACK, u"Artículo destacado"), 'et': (CAT, u"Eeskujulikud artiklid"), 'eu': (CAT, u"Nabarmendutako artikuluak"), - 'ext':(BACK,u"Destacau"), - 'fa': (BACK,u"مقاله برگزیده"), + 'ext': (BACK, u"Destacau"), + 'fa': (BACK, u"مقاله برگزیده"), 'fi': (CAT, u"Suositellut sivut"), 'fo': (CAT, u"Mánaðargrein"), 'fr': (CAT, u"Article de qualité"), - 'frr':(BACK,u"Exzellent"), + 'frr': (BACK, u"Exzellent"), 'gl': (CAT, u"Wikipedia:Artigos de calidade"), 'gv': (CAT, u"Artyn reiht"), 'he': (CAT, u"ערכים מומלצים"), - 'hi': (BACK,u"निर्वाचित लेख"), + 'hi': (BACK, u"निर्वाचित लेख"), 'hr': (CAT, u"Izabrani članci"), - 'hsb':(CAT, u"Ekscelentny"), + 'hsb': (CAT, u"Ekscelentny"), 'hu': (CAT, u"Kiemelt cikkek"), - 'hy': (BACK,u"Ընտրված հոդված"), + 'hy': (BACK, u"Ընտրված հոդված"), 'ia': (CAT, u"Wikipedia:Articulos eminente"), 'id': (BACK, u"Featured article"), 'is': (CAT, u"Wikipedia:Úrvalsgreinar"), 'it': (CAT, u"Voci in vetrina"), - 'ja': (BACK,u"Featured article"), + 'ja': (BACK, u"Featured article"), 'ka': (CAT, u"რჩეული სტატიები"), 'kk': (CAT, u"Уикипедия:Таңдаулы мақалалар"), 'kl': (CAT, u"Anbefalet"), - 'km': (BACK,u"អត្ថបទពិសេស"), - 'kn': (BACK,u"ವಿಶೇಷ ಲೇಖನ"), + 'km': (BACK, u"អត្ថបទពិសេស"), + 'kn': (BACK, u"ವಿಶೇಷ ಲೇಖನ"), 'ko': (CAT, u"알찬 글"), - 'krc':(CAT, u"Википедия:Сайланнган статьяла"), + 'krc': (CAT, u"Википедия:Сайланнган статьяла"), 'kv': (CAT, u"Википедия:Бур гижӧдъяс"), 'la': (CAT, u"Paginae mensis"), - 'lad':(CAT, u"Artikolos valutosos"), + 'lad': (CAT, u"Artikolos valutosos"), 'li': (CAT, u"Wikipedia:Sjterartikele"), - 'lmo':(CAT, u"Articol ben faa"), + 'lmo': (CAT, u"Articol ben faa"), 'lo': (CAT, u"ບົດຄວາມດີເດັ່ນ"), 'lt': (CAT, u"Vikipedijos pavyzdiniai straipsniai"), 'lv': (CAT, u"Vērtīgi raksti"), 'mk': (BACK, u"Избрана"), - 'ml': (BACK,u"Featured"), + 'ml': (BACK, u"Featured"), 'mt': (CAT, u"Artikli fil-vetrina"), 'mr': (CAT, u"मुखपृष्ठ सदर लेख"), - 'ms': (BACK,u"Rencana pilihan"), - 'nah':(BACK,u"Featured article"), + 'ms': (BACK, u"Rencana pilihan"), + 'nah': (BACK, u"Featured article"), 'nds-nl': (BACK, u"Etelazie"), 'nl': (CAT, u"Wikipedia:Etalage-artikelen"), - 'nn': (BACK,u"God artikkel"), + 'nn': (BACK, u"God artikkel"), 'no': (CAT, u"Utmerkede artikler"), 'nv': (CAT, u"Naaltsoos nizhónígo ályaaígíí"), 'oc': (CAT, u"Article de qualitat"), @@ -204,33 +209,33 @@ 'qu': (CAT, u"Wikipidiya:Kusa qillqa"), 'ro': (CAT, u"Articole de calitate"), 'ru': (BACK, u"Избранная статья"), - 'sco':(CAT, u"Featurt"), + 'sco': (CAT, u"Featurt"), 'sh': (CAT, u"Izabrani članci"), 'simple': (CAT, u"Very good articles"), - 'sk': (BACK,u"Perfektný článok"), + 'sk': (BACK, u"Perfektný článok"), 'sl': (CAT, u"Vsi izbrani članki"), - 'sq': (BACK,u"Artikulli perfekt"), + 'sq': (BACK, u"Artikulli perfekt"), 'sr': (CAT, u"Изабрани"), 'sv': (CAT, u"Wikipedia:Utmärkta artiklar"), - 'sw': (BACK,u"Makala_nzuri_sana"), - 'szl':(CAT, u"Wyrůżńůne artikle"), + 'sw': (BACK, u"Makala_nzuri_sana"), + 'szl': (CAT, u"Wyrůżńůne artikle"), 'ta': (CAT, u"சிறப்புக் கட்டுரைகள்"), 'te': (CAT, u"విశేషవ్యాసాలు"), - 'th': (BACK,u"บทความคัดสรร"), - 'tl': (BACK,u"Napiling artikulo"), + 'th': (BACK, u"บทความคัดสรร"), + 'tl': (BACK, u"Napiling artikulo"), 'tn': (CAT, u"Featured articles"), - 'tr': (BACK,u"Seçkin madde"), + 'tr': (BACK, u"Seçkin madde"), 'tt': (CAT, u"Сайланган мәкаләләр"), - 'udm':(CAT, u"Википедия:Быръем статьяос"), + 'udm': (CAT, u"Википедия:Быръем статьяос"), 'uk': (CAT, u"Вибрані статті"), 'ur': (CAT, u"منتخب مقالے"), 'uz': (CAT, u"Vikipediya:Tanlangan maqolalar"), - 'vec':(BACK,u"Vetrina"), + 'vec': (BACK, u"Vetrina"), 'vi': (CAT, u"Bài viết chọn lọc"), 'vo': (CAT, u"Yegeds gudik"), 'wa': (CAT, u"Raspepyî årtike"), 'yi': (CAT, u"רעקאמענדירטע ארטיקלען"), - 'yo': (BACK,u"Àyọkà pàtàkì"), + 'yo': (BACK, u"Àyọkà pàtàkì"), 'zh': (CAT, u"特色条目"), 'zh-classical': (CAT, u"卓著"), 'zh-yue': (BACK, u"正文"), @@ -242,7 +247,7 @@ 'cs': (CAT, u"Wikipedie:Dobré články"), 'da': (CAT, u"Gode artikler"), 'de': (CAT, u"Wikipedia:Lesenswert"), - #'dsb':(CAT, u"Naraźenje za pógódnośenje"), +## 'dsb': (CAT, u"Naraźenje za pógódnośenje"), 'en': (CAT, u"Wikipedia good articles"), 'eo': (CAT, u"Legindaj artikoloj"), 'es': (CAT, u"Wikipedia:Artículos buenos"), @@ -250,12 +255,12 @@ 'fa': (CAT, u"مقاله‌های خوب"), 'fi': (CAT, u"Hyvät artikkelit"), 'fr': (CAT, u"Bon article"), - 'hsb':(CAT, u"Namjet za pohódnoćenje"), - 'id': (BACK,u"Artikel bagus"), + 'hsb': (CAT, u"Namjet za pohódnoćenje"), + 'id': (BACK, u"Artikel bagus"), 'is': (CAT, u"Wikipedia:Gæðagreinar"), - 'ja': (BACK,u"Good article"), + 'ja': (BACK, u"Good article"), 'ko': (CAT, u"좋은 글"), - 'ksh':(CAT, u"Joode Aatikkel"), + 'ksh': (CAT, u"Joode Aatikkel"), 'lt': (CAT, u"Vertingi straipsniai"), 'lv': (CAT, u"Labi raksti"), 'no': (CAT, u"Anbefalte artikler"), @@ -265,9 +270,9 @@ 'ro': (BACK, u"Articol bun"), 'ru': (CAT, u"Википедия:Хорошие статьи по алфавиту"), 'simple': (CAT, u"Good articles"), - 'sr': (BACK,u"Иконица добар"), + 'sr': (BACK, u"Иконица добар"), 'sv': (CAT, u"Wikipedia:Bra artiklar"), - 'tr': (BACK,u"Kaliteli madde"), + 'tr': (BACK, u"Kaliteli madde"), 'uk': (CAT, u"Вікіпедія:Добрі статті"), 'uz': (CAT, u"Vikipediya:Yaxshi maqolalar"), 'yi': (CAT, u"וויקיפעדיע גוטע ארטיקלען"), @@ -283,7 +288,7 @@ 'fa': (BACK, u"فهرست برگزیده"), 'id': (BACK, u'Featured list'), 'ja': (BACK, u'Featured List'), - 'ksh':(CAT, u"Joode Leß"), + 'ksh': (CAT, u"Joode Leß"), 'no': (BACK, u'God liste'), 'pl': (BACK, u'Medalista'), 'pt': (BACK, u'Anexo destacado'), @@ -295,7 +300,8 @@ 'zh': (BACK, u'特色列表'), } -# Third parameter is the sort key indicating articles to hide from the given list +# Third parameter is the sort key indicating articles to hide from the given +# list former_name = { 'ca': (CAT, u"Arxiu de propostes de la retirada de la distinció"), 'en': (CAT, u"Wikipedia former featured articles", "#"), @@ -310,8 +316,9 @@ 'zh': (CAT, u"已撤销的特色条目"), } + def featuredArticles(site, pType): - articles=[] + articles = [] if pType == 'good': info = good_name elif pType == 'former': @@ -335,10 +342,10 @@ hide = None raw = method(site, name, hide) for p in raw: - if p.namespace() == 0: # Article + if p.namespace() == 0: # Article articles.append(p) # Article talk (like in English) - elif p.namespace() == 1 and site.lang <> 'el': + elif p.namespace() == 1 and site.lang != 'el': articles.append(pywikibot.Page(p.site(), p.title(withNamespace=False))) pywikibot.output( @@ -347,30 +354,34 @@ for p in articles: yield copy(p) + def findTranslated(page, oursite=None, quiet=False): if not oursite: - oursite=pywikibot.getSite() + oursite = pywikibot.getSite() if page.isRedirectPage(): page = page.getRedirectTarget() try: - iw=page.interwiki() + iw = page.interwiki() except: pywikibot.output(u"%s -> no interwiki, giving up" % page.title()) - return None - ourpage=None + return + + ourpage = None for p in iw: - if p.site()==oursite: - ourpage=p + if p.site() == oursite: + ourpage = p break if not ourpage: if not quiet: pywikibot.output(u"%s -> no corresponding page in %s" % (page.title(), oursite)) - return None + return + if not ourpage.exists(): pywikibot.output(u"%s -> our page doesn't exist: %s" % (page.title(), ourpage.title())) - return None + return + if ourpage.isRedirectPage(): ourpage = ourpage.getRedirectTarget() pywikibot.output(u"%s -> corresponding page is %s" @@ -378,25 +389,31 @@ if ourpage.namespace() != 0: pywikibot.output(u"%s -> not in the main namespace, skipping" % page.title()) - return None + return + if ourpage.isRedirectPage(): pywikibot.output(u"%s -> double redirect, skipping" % page.title()) - return None + return + if not ourpage.exists(): - pywikibot.output(u"%s -> page doesn't exist, skipping" % ourpage.title()) - return None + pywikibot.output(u"%s -> page doesn't exist, skipping" + % ourpage.title()) + return + try: iw = ourpage.interwiki() except: - return None - backpage=None + return + + backpage is None for p in iw: if p.site() == page.site(): backpage = p break if not backpage: pywikibot.output(u"%s -> no back interwiki ref" % page.title()) - return None + return + if backpage == page: # everything is ok return ourpage @@ -407,9 +424,9 @@ return ourpage pywikibot.output(u"%s -> back interwiki ref target is %s" % (page.title(), backpage.title())) - return None -def getTemplateList (lang, pType): + +def getTemplateList(lang, pType): if pType == 'good': try: templates = template_good[lang] @@ -422,7 +439,7 @@ templates += template_lists['_default'] except KeyError: templates = template_lists['_default'] - else: #pType in ['former', 'featured'] + else: # pType in ['former', 'featured'] try: templates = template[lang] templates += template['_default'] @@ -430,6 +447,7 @@ templates = template['_default'] return templates + def featuredWithInterwiki(fromsite, tosite, template_on_top, pType, quiet, dry=False): if not fromsite.lang in cache: @@ -438,36 +456,39 @@ cache[fromsite.lang][tosite.lang] = {} cc = cache[fromsite.lang][tosite.lang] if nocache: - cc={} + cc = {} templatelist = getTemplateList(tosite.lang, pType) findtemplate = '(' + '|'.join(templatelist) + ')' - re_Link_FA=re.compile(ur"\{\{%s\|%s\}\}" - % (findtemplate.replace(u' ', u'[ _]'), - fromsite.lang), re.IGNORECASE) + re_Link_FA = re.compile(ur"\{\{%s\|%s\}\}" + % (findtemplate.replace(u' ', u'[ _]'), + fromsite.lang), re.IGNORECASE) gen = featuredArticles(fromsite, pType) gen = PreloadingGenerator(gen) - - pairs=[] + pairs = [] for a in gen: if a.title() < afterpage: continue + if u"/" in a.title() and a.namespace() != 0: pywikibot.output(u"%s is a subpage" % a.title()) continue + if a.title() in cc: - pywikibot.output(u"(cached) %s -> %s"%(a.title(), cc[a.title()])) + pywikibot.output(u"(cached) %s -> %s" % (a.title(), cc[a.title()])) continue + if a.isRedirectPage(): - a=a.getRedirectTarget() + a = a.getRedirectTarget() try: if not a.exists(): pywikibot.output(u"source page doesn't exist: %s" % a.title()) continue + atrans = findTranslated(a, tosite, quiet) - if pType!='former': + if pType != 'former': if atrans: - text=atrans.get() - m=re_Link_FA.search(text) + text = atrans.get() + m = re_Link_FA.search(text) if m: pywikibot.output(u"(already done)") else: @@ -475,14 +496,13 @@ if (not interactive or pywikibot.input( u'Connecting %s -> %s. Proceed? [Y/N]' - % (a.title(), atrans.title())) in ['Y', 'y'] - ): + % (a.title(), atrans.title())) in ['Y', 'y']): site = pywikibot.getSite() comment = pywikibot.setAction( i18n.twtranslate(site, 'featured-' + pType, {'page': unicode(a)})) ### Moving {{Link FA|xx}} to top of interwikis ### - if template_on_top == True: + if template_on_top: # Getting the interwiki iw = pywikibot.getLanguageLinks(text, site) # Removing the interwiki @@ -493,35 +513,35 @@ text = pywikibot.replaceLanguageLinks(text, iw, site) - ### Placing {{Link FA|xx}} right next to corresponding interwiki ### + # Placing {{Link FA|xx}} right next to + # corresponding interwiki else: - text=(text[:m.end()] - + (u" {{%s|%s}}" % (templatelist[0], - fromsite.lang)) - + text[m.end():]) + text = (text[:m.end()] + + (u" {{%s|%s}}" % (templatelist[0], + fromsite.lang)) + + text[m.end():]) if not dry: try: atrans.put(text, comment) except pywikibot.LockedPage: pywikibot.output(u'Page %s is locked!' % atrans.title()) - cc[a.title()]=atrans.title() + cc[a.title()] = atrans.title() else: if atrans: - text=atrans.get() - m=re_Link_FA.search(text) + text = atrans.get() + m = re_Link_FA.search(text) if m: # insert just before interwiki if (not interactive or pywikibot.input( u'Connecting %s -> %s. Proceed? [Y/N]' - % (a.title(), atrans.title())) in ['Y', 'y'] - ): + % (a.title(), atrans.title())) in ['Y', 'y']): site = pywikibot.getSite() comment = pywikibot.setAction( i18n.twtranslate(site, 'featured-former', {'page': unicode(a)})) - text = re.sub(re_Link_FA,'',text) + text = re.sub(re_Link_FA, '', text) if not dry: try: atrans.put(text, comment) @@ -530,10 +550,11 @@ % atrans.title()) else: pywikibot.output(u"(already done)") - cc[a.title()]=atrans.title() + cc[a.title()] = atrans.title() except pywikibot.PageNotSaved, e: pywikibot.output(u"Page not saved") + def main(*args): global nocache, interactive, afterpage, cache nocache = 0 @@ -543,23 +564,23 @@ template_on_top = True featuredcount = False - fromlang=[] + fromlang = [] processType = 'featured' doAll = False - part = False + part = False quiet = False for arg in pywikibot.handleArgs(): if arg == '-interactive': - interactive=1 + interactive = 1 elif arg == '-nocache': - nocache=1 + nocache = 1 elif arg.startswith('-fromlang:'): - fromlang=arg[10:].split(",") + fromlang = arg[10:].split(",") part = True elif arg == '-fromall': doAll = True elif arg.startswith('-after:'): - afterpage=arg[7:] + afterpage = arg[7:] elif arg == '-side': template_on_top = False elif arg == '-count': @@ -578,8 +599,10 @@ # BUG: range with zh-min-nan (3 "-") if len(fromlang) == 1 and fromlang[0].index("-") >= 0: start, end = fromlang[0].split("--", 1) - if not start: start = "" - if not end: end = "zzzzzzz" + if not start: + start = "" + if not end: + end = "zzzzzzz" if processType == 'good': fromlang = [lang for lang in good_name.keys() if lang >= start and lang <= end] @@ -605,11 +628,11 @@ else: fromlang = featured_name.keys() - filename="cache/" + processType + filename = "cache/" + processType try: - cache=pickle.load(file(filename,"rb")) + cache = pickle.load(file(filename, "rb")) except: - cache={} + cache = {} if not fromlang: pywikibot.showHelp('featured') @@ -625,6 +648,7 @@ if t.exists(): hasTemplate = True break + try: for ll in fromlang: fromsite = pywikibot.getSite(ll) @@ -639,16 +663,18 @@ % (processType, pywikibot.getSite().lang)) pywikibot.output('Quitting program...') break - elif fromsite != pywikibot.getSite(): + elif fromsite != pywikibot.getSite(): featuredWithInterwiki(fromsite, pywikibot.getSite(), template_on_top, processType, quiet, - pywikibot.simulate) + config.simulate) except KeyboardInterrupt: pywikibot.output('\nQuitting program...') finally: if not nocache: - pickle.dump(cache,file(filename,"wb")) + return + pickle.dump(cache, file(filename, "wb")) + if __name__ == "__main__": try: main()

1 0

SVN: [11720] branches/rewrite/scripts/featured.py
by xqt＠svn.wikimedia.org 07 Jul '13

07 Jul '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11720 Revision: 11720 Author: xqt Date: 2013-07-07 15:48:09 +0000 (Sun, 07 Jul 2013) Log Message: ----------- copied from trunk r11558 Added Paths: ----------- branches/rewrite/scripts/featured.py Copied: branches/rewrite/scripts/featured.py (from rev 11718, trunk/pywikipedia/featured.py) =================================================================== --- branches/rewrite/scripts/featured.py (rev 0) +++ branches/rewrite/scripts/featured.py 2013-07-07 15:48:09 UTC (rev 11720) @@ -0,0 +1,656 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" +This script understands various command-line arguments: + +-interactive: ask before changing each page + +-nocache doesn't include /cache/featured /cache/lists or /cache/good + file to remember if the article already was verified. + +-fromlang:xx,yy xx,yy,zz,.. are the languages to be verified. +-fromlang:ar--fi Another possible with range the languages + +-fromall to verify all languages. + +-after:zzzz process pages after and including page zzzz + +-side use -side if you want to move all {{Link FA|lang}} next to the + corresponding interwiki links. Default is placing + {{Link FA|lang}} on top of the interwiki links. + +-count Only counts how many featured/good articles exist + on all wikis (given with the "-fromlang" argument) or + on several language(s) (when using the "-fromall" argument). + Example: featured.py -fromlang:en,he -count + counts how many featured articles exist in the en and he + wikipedias. + +-lists use this script for featured lists. + +-good use this script for good articles. + +-former use this script for removing {{Link FA|xx}} from former + fearured articles + +-quiet no corresponding pages are displayed. + +usage: featured.py [-interactive] [-nocache] [-top] [-after:zzzz] [-fromlang:xx,yy--zz|-fromall] + +""" +__version__ = '$Id$' + +# +# (C) Maxim Razin, 2005 +# (C) Leonardo Gregianin, 2005-2008 +# (C) xqt, 2009-2012 +# (C) Pywikipedia bot team, 2005-2012 +# +# Distributed under the terms of the MIT license. +# + +import sys, re, pickle, os.path +from copy import copy +import wikipedia as pywikibot +from pywikibot import i18n +import catlib, config +from pagegenerators import PreloadingGenerator + +def CAT(site, name, hide): + name = site.namespace(14) + ':' + name + cat=catlib.Category(site, name) + for article in cat.articles(endsort=hide): + yield article + if hide: + for article in cat.articles(startFrom=unichr(ord(hide)+1)): + yield article + +def BACK(site, name, hide): + p=pywikibot.Page(site, name, defaultNamespace=10) + return [page for page in p.getReferences(follow_redirects=False, + onlyTemplateInclusion=True)] + +# ALL wikis use 'Link FA', and sometimes other localized templates. +# We use _default AND the localized ones +template = { + '_default': ['Link FA'], + 'als': ['LinkFA'], + 'an': ['Destacato', 'Destacau'], + 'ar': [u'وصلة مقالة مختارة'], + 'ast':['Enllaz AD'], + 'az': ['Link FM'], + 'br': ['Liamm PuB', 'Lien AdQ'], + 'ca': [u'Enllaç AD', 'Destacat'], + 'cy': ['Cyswllt erthygl ddethol', 'Dolen ED'], + 'eo': ['LigoElstara'], + 'en': ['Link FA', 'FA link'], + 'es': ['Destacado'], + 'eu': ['NA lotura'], + 'fr': ['Lien AdQ'], + 'fur':['Leam VdC'], + 'ga': ['Nasc AR'], + 'hi': ['Link FA', 'Lien AdQ'], + 'is': [u'Tengill ÚG'], + 'it': ['Link AdQ'], + 'no': ['Link UA'], + 'oc': ['Ligam AdQ', 'Lien AdQ'], + 'ro': [u'Legătură AC', u'Legătură AF'], + 'sv': ['UA', 'Link UA'], + 'tr': ['Link SM'], + 'vi': [u'Liên kết chọn lọc'], + 'vo': [u'Yüm YG'], + 'yi': [u'רא'], +} + +template_good = { + '_default': ['Link GA'], + 'ar': [u'وصلة مقالة جيدة'], + 'da': ['Link GA', 'Link AA'], + 'eo': ['LigoLeginda'], + 'es': ['Bueno'], + 'fr': ['Lien BA'], + 'is': ['Tengill GG'], + 'it': ['Link VdQ'], + 'nn': ['Link AA'], + 'no': ['Link AA'], + 'pt': ['Bom interwiki'], + #'tr': ['Link GA', 'Link KM'], + 'vi': [u'Liên kết bài chất lượng tốt'], + 'wo': ['Lien BA'], +} + +template_lists = { + '_default': ['Link FL'], + 'no': ['Link GL'], +} + +featured_name = { + 'af': (BACK,u"Voorbladster"), + 'als':(CAT, u"Wikipedia:Bsunders glungener Artikel"), + 'am': (CAT, u"Wikipedia:Featured article"), + 'an': (CAT, u"Articlos destacatos"), + 'ang':(CAT, u"Fulgōd ȝeƿritu"), + 'ar': (CAT, u"مقالات مختارة"), + 'ast':(CAT, u"Uiquipedia:Artículos destacaos"), + 'az': (BACK,u"Seçilmiş məqalə"), + 'bar':(CAT, u"Berig"), + 'bat-smg': (CAT, u"Vikipedėjės pavīzdėnē straipsnē"), + 'be-x-old':(CAT, u"Вікіпэдыя:Абраныя артыкулы"), + 'bg': (CAT, u"Избрани статии"), + 'bn': (BACK,u"নির্বাচিত নিবন্ধ"), + 'br': (CAT, u"Pennadoù eus an dibab"), + 'bs': (CAT, u"Odabrani članci"), + 'ca': (CAT, u"Llista d'articles de qualitat"), + 'ceb':(CAT, u"Mga napiling artikulo"), + 'cs': (CAT, u"Wikipedie:Nejlepší články"), + 'cy': (BACK,u"Erthygl ddethol"), + 'da': (CAT, u"Fremragende artikler"), + 'de': (CAT, u"Wikipedia:Exzellent"), + 'dv': (BACK, u"Featured article"), + 'el': (BACK,u"Αξιόλογο άρθρο"), + 'en': (CAT, u"Featured articles"), + 'eo': (CAT, u"Elstaraj artikoloj"), + 'es': (BACK, u"Artículo destacado"), + 'et': (CAT, u"Eeskujulikud artiklid"), + 'eu': (CAT, u"Nabarmendutako artikuluak"), + 'ext':(BACK,u"Destacau"), + 'fa': (BACK,u"مقاله برگزیده"), + 'fi': (CAT, u"Suositellut sivut"), + 'fo': (CAT, u"Mánaðargrein"), + 'fr': (CAT, u"Article de qualité"), + 'frr':(BACK,u"Exzellent"), + 'gl': (CAT, u"Wikipedia:Artigos de calidade"), + 'gv': (CAT, u"Artyn reiht"), + 'he': (CAT, u"ערכים מומלצים"), + 'hi': (BACK,u"निर्वाचित लेख"), + 'hr': (CAT, u"Izabrani članci"), + 'hsb':(CAT, u"Ekscelentny"), + 'hu': (CAT, u"Kiemelt cikkek"), + 'hy': (BACK,u"Ընտրված հոդված"), + 'ia': (CAT, u"Wikipedia:Articulos eminente"), + 'id': (BACK, u"Featured article"), + 'is': (CAT, u"Wikipedia:Úrvalsgreinar"), + 'it': (CAT, u"Voci in vetrina"), + 'ja': (BACK,u"Featured article"), + 'ka': (CAT, u"რჩეული სტატიები"), + 'kk': (CAT, u"Уикипедия:Таңдаулы мақалалар"), + 'kl': (CAT, u"Anbefalet"), + 'km': (BACK,u"អត្ថបទពិសេស"), + 'kn': (BACK,u"ವಿಶೇಷ ಲೇಖನ"), + 'ko': (CAT, u"알찬 글"), + 'krc':(CAT, u"Википедия:Сайланнган статьяла"), + 'kv': (CAT, u"Википедия:Бур гижӧдъяс"), + 'la': (CAT, u"Paginae mensis"), + 'lad':(CAT, u"Artikolos valutosos"), + 'li': (CAT, u"Wikipedia:Sjterartikele"), + 'lmo':(CAT, u"Articol ben faa"), + 'lo': (CAT, u"ບົດຄວາມດີເດັ່ນ"), + 'lt': (CAT, u"Vikipedijos pavyzdiniai straipsniai"), + 'lv': (CAT, u"Vērtīgi raksti"), + 'mk': (BACK, u"Избрана"), + 'ml': (BACK,u"Featured"), + 'mt': (CAT, u"Artikli fil-vetrina"), + 'mr': (CAT, u"मुखपृष्ठ सदर लेख"), + 'ms': (BACK,u"Rencana pilihan"), + 'nah':(BACK,u"Featured article"), + 'nds-nl': (BACK, u"Etelazie"), + 'nl': (CAT, u"Wikipedia:Etalage-artikelen"), + 'nn': (BACK,u"God artikkel"), + 'no': (CAT, u"Utmerkede artikler"), + 'nv': (CAT, u"Naaltsoos nizhónígo ályaaígíí"), + 'oc': (CAT, u"Article de qualitat"), + 'pl': (CAT, u"Artykuły na medal"), + 'pt': (CAT, u"!Artigos destacados"), + 'qu': (CAT, u"Wikipidiya:Kusa qillqa"), + 'ro': (CAT, u"Articole de calitate"), + 'ru': (BACK, u"Избранная статья"), + 'sco':(CAT, u"Featurt"), + 'sh': (CAT, u"Izabrani članci"), + 'simple': (CAT, u"Very good articles"), + 'sk': (BACK,u"Perfektný článok"), + 'sl': (CAT, u"Vsi izbrani članki"), + 'sq': (BACK,u"Artikulli perfekt"), + 'sr': (CAT, u"Изабрани"), + 'sv': (CAT, u"Wikipedia:Utmärkta artiklar"), + 'sw': (BACK,u"Makala_nzuri_sana"), + 'szl':(CAT, u"Wyrůżńůne artikle"), + 'ta': (CAT, u"சிறப்புக் கட்டுரைகள்"), + 'te': (CAT, u"విశేషవ్యాసాలు"), + 'th': (BACK,u"บทความคัดสรร"), + 'tl': (BACK,u"Napiling artikulo"), + 'tn': (CAT, u"Featured articles"), + 'tr': (BACK,u"Seçkin madde"), + 'tt': (CAT, u"Сайланган мәкаләләр"), + 'udm':(CAT, u"Википедия:Быръем статьяос"), + 'uk': (CAT, u"Вибрані статті"), + 'ur': (CAT, u"منتخب مقالے"), + 'uz': (CAT, u"Vikipediya:Tanlangan maqolalar"), + 'vec':(BACK,u"Vetrina"), + 'vi': (CAT, u"Bài viết chọn lọc"), + 'vo': (CAT, u"Yegeds gudik"), + 'wa': (CAT, u"Raspepyî årtike"), + 'yi': (CAT, u"רעקאמענדירטע ארטיקלען"), + 'yo': (BACK,u"Àyọkà pàtàkì"), + 'zh': (CAT, u"特色条目"), + 'zh-classical': (CAT, u"卓著"), + 'zh-yue': (BACK, u"正文"), +} + +good_name = { + 'ar': (CAT, u"مقالات جيدة"), + 'ca': (CAT, u"Llista d'articles bons"), + 'cs': (CAT, u"Wikipedie:Dobré články"), + 'da': (CAT, u"Gode artikler"), + 'de': (CAT, u"Wikipedia:Lesenswert"), + #'dsb':(CAT, u"Naraźenje za pógódnośenje"), + 'en': (CAT, u"Wikipedia good articles"), + 'eo': (CAT, u"Legindaj artikoloj"), + 'es': (CAT, u"Wikipedia:Artículos buenos"), + 'et': (CAT, u"Head artiklid"), + 'fa': (CAT, u"مقاله‌های خوب"), + 'fi': (CAT, u"Hyvät artikkelit"), + 'fr': (CAT, u"Bon article"), + 'hsb':(CAT, u"Namjet za pohódnoćenje"), + 'id': (BACK,u"Artikel bagus"), + 'is': (CAT, u"Wikipedia:Gæðagreinar"), + 'ja': (BACK,u"Good article"), + 'ko': (CAT, u"좋은 글"), + 'ksh':(CAT, u"Joode Aatikkel"), + 'lt': (CAT, u"Vertingi straipsniai"), + 'lv': (CAT, u"Labi raksti"), + 'no': (CAT, u"Anbefalte artikler"), + 'oc': (CAT, u"Bon article"), + 'pl': (CAT, u"Dobre artykuły"), + 'pt': (CAT, u"Artigos bons"), + 'ro': (BACK, u"Articol bun"), + 'ru': (CAT, u"Википедия:Хорошие статьи по алфавиту"), + 'simple': (CAT, u"Good articles"), + 'sr': (BACK,u"Иконица добар"), + 'sv': (CAT, u"Wikipedia:Bra artiklar"), + 'tr': (BACK,u"Kaliteli madde"), + 'uk': (CAT, u"Вікіпедія:Добрі статті"), + 'uz': (CAT, u"Vikipediya:Yaxshi maqolalar"), + 'yi': (CAT, u"וויקיפעדיע גוטע ארטיקלען"), + 'zh': (CAT, u"優良條目"), + 'zh-classical': (CAT, u"正典"), +} + +lists_name = { + 'ar': (BACK, u'قائمة مختارة'), + 'da': (BACK, u'FremragendeListe'), + 'de': (BACK, u'Informativ'), + 'en': (BACK, u'Featured list'), + 'fa': (BACK, u"فهرست برگزیده"), + 'id': (BACK, u'Featured list'), + 'ja': (BACK, u'Featured List'), + 'ksh':(CAT, u"Joode Leß"), + 'no': (BACK, u'God liste'), + 'pl': (BACK, u'Medalista'), + 'pt': (BACK, u'Anexo destacado'), + 'ro': (BACK, u'Listă de calitate'), + 'ru': (BACK, u'Избранный список или портал'), + 'tr': (BACK, u'Seçkin liste'), + 'uk': (BACK, u'Вибраний список'), + 'vi': (BACK, u'Sao danh sách chọn lọc'), + 'zh': (BACK, u'特色列表'), +} + +# Third parameter is the sort key indicating articles to hide from the given list +former_name = { + 'ca': (CAT, u"Arxiu de propostes de la retirada de la distinció"), + 'en': (CAT, u"Wikipedia former featured articles", "#"), + 'es': (CAT, u"Wikipedia:Artículos anteriormente destacados"), + 'fa': (CAT, u"مقاله‌های برگزیده پیشین"), + 'hu': (CAT, u"Korábbi kiemelt cikkek"), + 'pl': (CAT, u"Byłe artykuły na medal"), + 'pt': (CAT, u"!Ex-Artigos_destacados"), + 'ru': (CAT, u"Википедия:Устаревшие избранные статьи"), + 'th': (CAT, u"บทความคัดสรรในอดีต"), + 'tr': (CAT, u"Vikipedi eski seçkin maddeler"), + 'zh': (CAT, u"已撤销的特色条目"), +} + +def featuredArticles(site, pType): + articles=[] + if pType == 'good': + info = good_name + elif pType == 'former': + info = former_name + elif pType == 'list': + info = lists_name + else: + info = featured_name + try: + method = info[site.lang][0] + except KeyError: + pywikibot.error( + u'language %s doesn\'t has %s category source.' + % (site.lang, pType)) + return + name = info[site.lang][1] + # hide #-sorted items on en-wiki + try: + hide = info[site.lang][2] + except IndexError: + hide = None + raw = method(site, name, hide) + for p in raw: + if p.namespace() == 0: # Article + articles.append(p) + # Article talk (like in English) + elif p.namespace() == 1 and site.lang <> 'el': + articles.append(pywikibot.Page(p.site(), + p.title(withNamespace=False))) + pywikibot.output( + '\03{lightred}** wikipedia:%s has %i %s articles\03{default}' + % (site.lang, len(articles), pType)) + for p in articles: + yield copy(p) + +def findTranslated(page, oursite=None, quiet=False): + if not oursite: + oursite=pywikibot.getSite() + if page.isRedirectPage(): + page = page.getRedirectTarget() + try: + iw=page.interwiki() + except: + pywikibot.output(u"%s -> no interwiki, giving up" % page.title()) + return None + ourpage=None + for p in iw: + if p.site()==oursite: + ourpage=p + break + if not ourpage: + if not quiet: + pywikibot.output(u"%s -> no corresponding page in %s" + % (page.title(), oursite)) + return None + if not ourpage.exists(): + pywikibot.output(u"%s -> our page doesn't exist: %s" + % (page.title(), ourpage.title())) + return None + if ourpage.isRedirectPage(): + ourpage = ourpage.getRedirectTarget() + pywikibot.output(u"%s -> corresponding page is %s" + % (page.title(), ourpage.title())) + if ourpage.namespace() != 0: + pywikibot.output(u"%s -> not in the main namespace, skipping" + % page.title()) + return None + if ourpage.isRedirectPage(): + pywikibot.output(u"%s -> double redirect, skipping" % page.title()) + return None + if not ourpage.exists(): + pywikibot.output(u"%s -> page doesn't exist, skipping" % ourpage.title()) + return None + try: + iw = ourpage.interwiki() + except: + return None + backpage=None + for p in iw: + if p.site() == page.site(): + backpage = p + break + if not backpage: + pywikibot.output(u"%s -> no back interwiki ref" % page.title()) + return None + if backpage == page: + # everything is ok + return ourpage + if backpage.isRedirectPage(): + backpage = backpage.getRedirectTarget() + if backpage == page: + # everything is ok + return ourpage + pywikibot.output(u"%s -> back interwiki ref target is %s" + % (page.title(), backpage.title())) + return None + +def getTemplateList (lang, pType): + if pType == 'good': + try: + templates = template_good[lang] + templates += template_good['_default'] + except KeyError: + templates = template_good['_default'] + elif pType == 'list': + try: + templates = template_lists[lang] + templates += template_lists['_default'] + except KeyError: + templates = template_lists['_default'] + else: #pType in ['former', 'featured'] + try: + templates = template[lang] + templates += template['_default'] + except KeyError: + templates = template['_default'] + return templates + +def featuredWithInterwiki(fromsite, tosite, template_on_top, pType, quiet, + dry=False): + if not fromsite.lang in cache: + cache[fromsite.lang] = {} + if not tosite.lang in cache[fromsite.lang]: + cache[fromsite.lang][tosite.lang] = {} + cc = cache[fromsite.lang][tosite.lang] + if nocache: + cc={} + templatelist = getTemplateList(tosite.lang, pType) + findtemplate = '(' + '|'.join(templatelist) + ')' + re_Link_FA=re.compile(ur"\{\{%s\|%s\}\}" + % (findtemplate.replace(u' ', u'[ _]'), + fromsite.lang), re.IGNORECASE) + gen = featuredArticles(fromsite, pType) + gen = PreloadingGenerator(gen) + + pairs=[] + for a in gen: + if a.title() < afterpage: + continue + if u"/" in a.title() and a.namespace() != 0: + pywikibot.output(u"%s is a subpage" % a.title()) + continue + if a.title() in cc: + pywikibot.output(u"(cached) %s -> %s"%(a.title(), cc[a.title()])) + continue + if a.isRedirectPage(): + a=a.getRedirectTarget() + try: + if not a.exists(): + pywikibot.output(u"source page doesn't exist: %s" % a.title()) + continue + atrans = findTranslated(a, tosite, quiet) + if pType!='former': + if atrans: + text=atrans.get() + m=re_Link_FA.search(text) + if m: + pywikibot.output(u"(already done)") + else: + # insert just before interwiki + if (not interactive or + pywikibot.input( + u'Connecting %s -> %s. Proceed? [Y/N]' + % (a.title(), atrans.title())) in ['Y', 'y'] + ): + site = pywikibot.getSite() + comment = pywikibot.setAction( + i18n.twtranslate(site, 'featured-' + pType, + {'page': unicode(a)})) + ### Moving {{Link FA|xx}} to top of interwikis ### + if template_on_top == True: + # Getting the interwiki + iw = pywikibot.getLanguageLinks(text, site) + # Removing the interwiki + text = pywikibot.removeLanguageLinks(text, site) + text += u"\r\n{{%s|%s}}\r\n" % (templatelist[0], + fromsite.lang) + # Adding the interwiki + text = pywikibot.replaceLanguageLinks(text, + iw, site) + + ### Placing {{Link FA|xx}} right next to corresponding interwiki ### + else: + text=(text[:m.end()] + + (u" {{%s|%s}}" % (templatelist[0], + fromsite.lang)) + + text[m.end():]) + if not dry: + try: + atrans.put(text, comment) + except pywikibot.LockedPage: + pywikibot.output(u'Page %s is locked!' + % atrans.title()) + cc[a.title()]=atrans.title() + else: + if atrans: + text=atrans.get() + m=re_Link_FA.search(text) + if m: + # insert just before interwiki + if (not interactive or + pywikibot.input( + u'Connecting %s -> %s. Proceed? [Y/N]' + % (a.title(), atrans.title())) in ['Y', 'y'] + ): + site = pywikibot.getSite() + comment = pywikibot.setAction( + i18n.twtranslate(site, 'featured-former', + {'page': unicode(a)})) + text = re.sub(re_Link_FA,'',text) + if not dry: + try: + atrans.put(text, comment) + except pywikibot.LockedPage: + pywikibot.output(u'Page %s is locked!' + % atrans.title()) + else: + pywikibot.output(u"(already done)") + cc[a.title()]=atrans.title() + except pywikibot.PageNotSaved, e: + pywikibot.output(u"Page not saved") + +def main(*args): + global nocache, interactive, afterpage, cache + nocache = 0 + interactive = 0 + afterpage = u"!" + cache = {} + + template_on_top = True + featuredcount = False + fromlang=[] + processType = 'featured' + doAll = False + part = False + quiet = False + for arg in pywikibot.handleArgs(): + if arg == '-interactive': + interactive=1 + elif arg == '-nocache': + nocache=1 + elif arg.startswith('-fromlang:'): + fromlang=arg[10:].split(",") + part = True + elif arg == '-fromall': + doAll = True + elif arg.startswith('-after:'): + afterpage=arg[7:] + elif arg == '-side': + template_on_top = False + elif arg == '-count': + featuredcount = True + elif arg == '-good': + processType = 'good' + elif arg == '-lists': + processType = 'list' + elif arg == '-former': + processType = 'former' + elif arg == '-quiet': + quiet = True + + if part: + try: + # BUG: range with zh-min-nan (3 "-") + if len(fromlang) == 1 and fromlang[0].index("-") >= 0: + start, end = fromlang[0].split("--", 1) + if not start: start = "" + if not end: end = "zzzzzzz" + if processType == 'good': + fromlang = [lang for lang in good_name.keys() + if lang >= start and lang <= end] + elif processType == 'list': + fromlang = [lang for lang in lists_name.keys() + if lang >= start and lang <= end] + elif processType == 'former': + fromlang = [lang for lang in former_name.keys() + if lang >= start and lang <= end] + else: + fromlang = [lang for lang in featured_name.keys() + if lang >= start and lang <= end] + except: + pass + + if doAll: + if processType == 'good': + fromlang = good_name.keys() + elif processType == 'list': + fromlang = lists_name.keys() + elif processType == 'former': + fromlang = former_name.keys() + else: + fromlang = featured_name.keys() + + filename="cache/" + processType + try: + cache=pickle.load(file(filename,"rb")) + except: + cache={} + + if not fromlang: + pywikibot.showHelp('featured') + sys.exit(1) + + fromlang.sort() + + #test whether this site has template enabled + hasTemplate = False + if not featuredcount: + for tl in getTemplateList(pywikibot.getSite().lang, processType): + t = pywikibot.Page(pywikibot.getSite(), u'Template:'+tl) + if t.exists(): + hasTemplate = True + break + try: + for ll in fromlang: + fromsite = pywikibot.getSite(ll) + if featuredcount: + try: + featuredArticles(fromsite, processType).next() + except StopIteration: + continue + elif not hasTemplate: + pywikibot.output( + u'\nNOTE: %s arcticles are not implemented at %s-wiki.' + % (processType, pywikibot.getSite().lang)) + pywikibot.output('Quitting program...') + break + elif fromsite != pywikibot.getSite(): + featuredWithInterwiki(fromsite, pywikibot.getSite(), + template_on_top, processType, quiet, + pywikibot.simulate) + except KeyboardInterrupt: + pywikibot.output('\nQuitting program...') + finally: + if not nocache: + pickle.dump(cache,file(filename,"wb")) + +if __name__ == "__main__": + try: + main() + finally: + pywikibot.stopme()

1 0

SVN: [11719] trunk/pywikipedia/externals/__init__.py
by xqt＠svn.wikimedia.org 07 Jul '13

07 Jul '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11719 Revision: 11719 Author: xqt Date: 2013-07-07 14:47:53 +0000 (Sun, 07 Jul 2013) Log Message: ----------- fix blocker due to unrecognized mime type introduced in r11524 Modified Paths: -------------- trunk/pywikipedia/externals/__init__.py Modified: trunk/pywikipedia/externals/__init__.py =================================================================== --- trunk/pywikipedia/externals/__init__.py 2013-07-07 09:43:39 UTC (rev 11718) +++ trunk/pywikipedia/externals/__init__.py 2013-07-07 14:47:53 UTC (rev 11719) @@ -23,7 +23,8 @@ # supports: 0. svn:externals # 1. package management system (yum, apt-get, ...) # 2. download from url (or svn, git repo) -# 3. checkout from mercurial repo ('hg clone ...' since url not available) +# 3. checkout from mercurial repo ('hg clone ...' since url not +# available) # (what about python eggs?!) # dependencies: (svn, python) # yum, apt-get or whatever your system uses @@ -32,7 +33,7 @@ modules_needed = { 'patch.exe': ({}, # for win32 only, unix/linux is already equipped with a patch tool { 'url': 'http://downloads.sourceforge.net/project/gnuwin32/patch/2.5.9-7/patch-2.5.9…', - 'path': 'bin/patch.exe',}), # OK + 'path': 'bin/patch.exe'}), # OK 'crontab': ({}, #{ 'url': 'https://github.com/josiahcarlson/parse-crontab/archive/master.zip', # 'path': 'parse-crontab-master/crontab',}), # OK @@ -43,11 +44,11 @@ #{ 'url': 'https://pypi.python.org/packages/source/o/odfpy/odfpy-0.9.6.tar.gz', # 'path': 'odfpy-0.9.6/odf',}), # OK { 'url': 'https://pypi.python.org/packages/source/o/odfpy/odfpy-0.9.4.tar.gz', - 'path': 'odfpy-0.9.4/odf',}, + 'path': 'odfpy-0.9.4/odf'}, {}), # OK 'openpyxl': ({}, { 'url': 'https://bitbucket.org/ericgazoni/openpyxl/get/1.5.6.tar.gz', - 'path': 'ericgazoni-openpyxl-e5934500ffac/openpyxl',}, + 'path': 'ericgazoni-openpyxl-e5934500ffac/openpyxl'}, {}), # OK # 'spelling': $ svn propedit svn:externals externals/. # spelling http://svn.wikimedia.org/svnroot/pywikipedia/trunk/spelling/ @@ -56,14 +57,14 @@ # 'simplejson': ({'linux-fedora': ['python-simplejson'], # 'linux-ubuntu': [''],}, 'BeautifulSoup.py': ({'linux-fedora': ['python-BeautifulSoup'], - 'linux-ubuntu': [''],}, + 'linux-ubuntu': ['']}, { 'url': 'https://pypi.python.org/packages/source/B/BeautifulSoup/BeautifulSoup-3.2.0…', 'path': 'BeautifulSoup-3.2.0/BeautifulSoup.py', #$ diff -Nau TEST_BeautifulSoup.py BeautifulSoup.py > patch-BeautifulSoup - 'patch': 'patch-BeautifulSoup',}, + 'patch': 'patch-BeautifulSoup'}, {}), # OK 'irclib': ({'linux-fedora': ['python-irclib'], - 'linux-ubuntu': [''],}, + 'linux-ubuntu': ['']}, {}, # http://python-irclib.sourceforge.net/ {}), # OK 'colormath': ({'linux-fedora': [], @@ -75,7 +76,7 @@ { 'url': 'http://vision.ece.ucsb.edu/segmentation/jseg/software/jseg.zip', 'path': 'jseg', #$ diff -Nau --exclude="*.o" --exclude="*.pyc" --exclude="segdist_cpp*" TEST_jseg/ jseg/ > patch-jseg - 'patch': 'patch-jseg',}, + 'patch': 'patch-jseg'}, {}), # OK 'jseg/jpeg-6b': ({}, { 'url': 'http://vision.ece.ucsb.edu/segmentation/jseg/software/jpeg-6b.zip', @@ -83,19 +84,19 @@ {}), # OK '_mlpy': ({}, { 'url': 'http://downloads.sourceforge.net/project/mlpy/mlpy%203.5.0/mlpy-3.5.0.tar.gz', - 'path': 'mlpy-3.5.0/mlpy',}, + 'path': 'mlpy-3.5.0/mlpy'}, {}), # OK '_music21': ({}, { 'url': 'http://music21.googlecode.com/files/music21-1.4.0.tar.gz', 'path': 'music21-1.4.0', #$ diff -Naur --exclude="*.pyc" TEST__music21/ _music21/ > patch-music21 - 'patch': 'patch-music21',}, + 'patch': 'patch-music21'}, {}), # OK # TODO: vvv (future; enable for and use in 'catimages.py', patch needed) '_ocropus': ({}, {}, { 'url': 'https://code.google.com/p/ocropus', - 'rev': 'ocropus-0.6',}), # OK + 'rev': 'ocropus-0.6'}), # OK # TODO: vvv (further clean-up and unlink - check with 'svn list') # 'opencv': $ svn propedit svn:externals externals/. # opencv https://svn.toolserver.org/svnroot/drtrigon/externals/opencv @@ -108,20 +109,20 @@ # createtrainsamples.pl https://svn.toolserver.org/svnroot/drtrigon/externals/haartraining/createtr… 'opencv/haarcascades': ({}, { 'url': 'https://svn.toolserver.org/svnroot/drtrigon/externals/haarcascades-full.tar…', - 'path': 'haarcascades',}, + 'path': 'haarcascades'}, {}), # OK # 'pdfminer' is not used anymore/at the moment... # 'pycolorname': $ svn propset svn:externals 'pycolorname https://svn.toolserver.org/svnroot/drtrigon/externals/pycolorname' externals/. 'pydmtx': ({'linux-fedora': ['python-libdmtx'], - 'linux-ubuntu': ['libdmtx-dev'],}, + 'linux-ubuntu': ['libdmtx-dev']}, { 'url': 'https://github.com/dmtx/dmtx-wrappers/archive/master.zip', 'path': 'dmtx-wrappers-master/python', #$ diff -Nau --exclude="*.pyc" TEST_pydmtx/ pydmtx/ > patch-pydmtx - 'patch': 'patch-pydmtx',}, + 'patch': 'patch-pydmtx'}, {}), # OK 'py_w3c': ({}, { 'url': 'https://bitbucket.org/nmb10/py_w3c/downloads/py_w3c-v0.1.0.tar.gz', - 'path': 'py_w3c-0.1.0/py_w3c',}, + 'path': 'py_w3c-0.1.0/py_w3c'}, {}), # OK # TODO: vvv (include) # 'TEST_slic': ({}, @@ -132,11 +133,11 @@ # 'path': 'slic-python-master',}), # OPEN # (2 download sources to same dir, compilation) + patch (at least for '__init__.py') needed '_zbar': ({'linux-fedora': ['zbar'], - 'linux-ubuntu': ['python-zbar'],}, + 'linux-ubuntu': ['python-zbar']}, { 'url': 'https://pypi.python.org/packages/source/z/zbar/zbar-0.10.tar.bz2', 'path': 'zbar-0.10', #$ diff -Nau --exclude="*.pyc" TEST__zbar/ _zbar/ > patch-zbar - 'patch': 'patch-zbar',}, + 'patch': 'patch-zbar'}, {}), # OK # TODO: vvv (include) # 'TEST__bob': ({}, @@ -157,12 +158,12 @@ modules_order = ['crontab', 'odf', 'openpyxl', 'BeautifulSoup.py', 'irclib', 'colormath', 'jseg', 'jseg/jpeg-6b', '_mlpy', '_music21', '_ocropus', 'opencv/haarcascades', 'pydmtx', 'py_w3c', - '_zbar',] + '_zbar', ] # OPEN: 'patch.py', 'opencv', 'slic', '_bob', 'xbob_flandmark', -import os, sys - +import os +import sys import wikipedia as pywikibot # sets externals path #from pywikibot.comms import http @@ -176,6 +177,7 @@ #return hasattr(sys.modules['wikipedia'], 'logger') return hasattr(pywikibot, 'logger') + # TODO: solve properly because this is just a work-a-round, because module # externals get imported in wikipedia.py before logger is setup properly, which # should be changed! (meanwhile this is acceptable because code here should be @@ -191,6 +193,7 @@ import platform return ("%s-%s" % (platform.system(), platform.dist()[0])).lower() + def show_question(which_files): lowlevel_warning("Required package missing: %s" % which_files) lowlevel_warning("A required package is missing, but externals can" @@ -237,10 +240,9 @@ lowlevel_warning("externals wants to install package(s) '%s'" % package_name) sucmd = "sudo %s" % cmd - result = os.system(sucmd) + return (result == 0) # 0 indicates success - return (result == 0) # 0 indicates success def linux_fedora_install(package_name): cmd = 'yum -y install' @@ -256,11 +258,10 @@ lowlevel_warning("externals wants to install package(s) '%s'" % package_name) sucmd = "su -c'%s'" % cmd - result = os.system(sucmd) - return (result == 0) + def linux_install(dependency_dictionary): """Tries to import a python module. If unsuccessful, tries to install the appropriate bundle and then reimport. py_import tries to be smart @@ -268,7 +269,7 @@ # Ugly fix to avoid circular import distro = guess_system() - if not dependency_dictionary.has_key(distro): + if not distro in dependency_dictionary: return False else: files = dependency_dictionary[distro] @@ -278,6 +279,7 @@ else: return False + # TODO: work this out... def windows_install(dependency_dictionary): return False @@ -289,7 +291,8 @@ if package: lowlevel_warning(u'Download package "%s" from %s' % (module, package['url'])) - import mimetypes, urllib2 + import mimetypes + import urllib2 for i in range(3): response = urllib2.urlopen(package['url']) #response = http.request(pywikibot.getSite(), package['url'], @@ -300,18 +303,23 @@ lowlevel_warning(u'Size of download: %s byte(s)' % response.headers['Content-Length']) #mime = response.headers['Content-Type'].lower().split('/') - mime = mimetypes.guess_type(package['url'], strict=True)[0].lower().split('/') + mime = mimetypes.guess_type(package['url'], + strict=True)[0].lower().split('/') lowlevel_warning(u'MIME type: %s' % mime) lowlevel_warning(u'Extract package "%s" to %s.' % (module, os.path.join(path, module))) if len(mime) > 1: - if mime[1] == 'zip': - import zipfile, StringIO + import StringIO + if mime[1] == 'zip' or mime[1] == 'x-zip-compressed': + import zipfile arch = zipfile.ZipFile(StringIO.StringIO(response.read())) elif mime[1] == 'x-tar': - import tarfile, StringIO + import tarfile arch = tarfile.open(fileobj=StringIO.StringIO(response.read())) + else: + raise NotImplementedError(u'Not implemented mime type %s' + % mime[1]) arch.extractall(os.path.join(path, '__setup_tmp/')) arch.close() import shutil @@ -324,37 +332,31 @@ lowlevel_warning(u'Install package "%s" by applying patch to %s.' % (module, os.path.join(path, module))) if sys.platform == 'win32': - cmd = '%s -p0 -d %s -i %s --binary' % (os.path.join(path, 'patch.exe'), - path, - os.path.join(path, package['patch'])) + cmd = '%s -p0 -d %s -i %s --binary' \ + % (os.path.join(path, 'patch.exe'), path, + os.path.join(path, package['patch'])) else: # unix/linux, (mac too?) - cmd = '%s -p0 -d %s < %s' % ('patch', - path, - os.path.join(path, package['patch'])) + cmd = '%s -p0 -d %s < %s' \ + % ('patch', path, + os.path.join(path, package['patch'])) result = os.system(cmd) lowlevel_warning(u'Package "%s" installed to %s.' % (module, os.path.join(path, module))) return (result == 0) - return False def mercurial_repo_install(package, module, path): if package: cmd = 'hg clone' - lowlevel_warning(u'Mercurial clone "%s" from %s' % (module, package['url'])) cmd += " -r %s %s %s" % (package['rev'], package['url'], os.path.join(path, module)) - result = os.system(cmd) - return (result == 0) - return False - def check_setup(m): path = os.path.dirname(os.path.abspath(os.path.join(os.curdir, __file__))) mf = os.path.join(path, m) @@ -374,8 +376,9 @@ if mercurial_repo_install(modules_needed[m][2], m, path): return - pywikibot.error(u'Package "%s" could not be found nor installed!' % m) + pywikibot.error(u'Package "%s" could not be found nor installed!' % m) + def check_setup_all(): #for m in modules_needed: for m in modules_order:

1 0

SVN: [11718] trunk/pywikipedia/catimages.py
by drtrigon＠svn.wikimedia.org 07 Jul '13

07 Jul '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11718 Revision: 11718 Author: drtrigon Date: 2013-07-07 09:43:39 +0000 (Sun, 07 Jul 2013) Log Message: ----------- bug fix; enable import of 'Pillow' instead of 'PIL', see http://fedoraproject.org/wiki/Features/Pillow improvement; enable usage of 'XMP' data from exiftool in order to get file history for categorization Modified Paths: -------------- trunk/pywikipedia/catimages.py Modified: trunk/pywikipedia/catimages.py =================================================================== --- trunk/pywikipedia/catimages.py 2013-07-07 09:39:57 UTC (rev 11717) +++ trunk/pywikipedia/catimages.py 2013-07-07 09:43:39 UTC (rev 11718) @@ -50,7 +50,11 @@ import re, urllib2, os, locale, sys, datetime, math, shutil, mimetypes, shelve import StringIO, json # fallback: simplejson from subprocess import Popen, PIPE -import Image, imghdr +try: + import Image # classic 'PIL' +except ImportError: + from PIL import Image # new 'PIL' fork 'Pillow' (fedora 19) +import imghdr #import ImageFilter scriptdir = os.path.dirname(sys.argv[0]) @@ -58,6 +62,7 @@ scriptdir = os.path.abspath(os.path.join(os.curdir, scriptdir)) # additional python packages (non-default but common) +sys.exc_clear() try: import numpy as np from scipy import ndimage, fftpack#, signal @@ -213,10 +218,6 @@ #'Dimensions': tuple(exif['ImageSize'].split(u'x')) if 'ImageSize' in exif else (None, None),} #'Mode': exif['ColorType'], } -# TODO: if '_detect_History' is not needed here, moveit back into _JpegFile !!! - #print "self._detect_History()" - #print self._detect_History() - ## https://pypi.python.org/pypi/hachoir-metadata (needs 'core' and 'parser') # #from hachoir_core.error import HachoirError @@ -333,46 +334,7 @@ return self._buffer_EXIF - def _detect_History(self): - res = self._util_get_DataTags_EXIF() - #a = [] - #for k in res.keys(): - # if 'history' in k.lower(): - # a.append( k ) - #for item in sorted(a): - # print item - # http://tilloy.net/dev/pyexiv2/api.html#pyexiv2.xmp.XmpTag - #print [getattr(res['Xmp.xmpMM.History'], item) for item in ['key', 'type', 'name', 'title', 'description', 'raw_value', 'value', ]] - result = [] - i = 1 - while (('Xmp.xmpMM.History[%i]' % i) in res): - data = { 'ID': i, - 'Software': u'-', - 'Timestamp': u'-', - 'Action': u'-', - 'Info': u'-', } - if ('Xmp.xmpMM.History[%i]/stEvt:softwareAgent'%i) in res: - data['Software'] = res['Xmp.xmpMM.History[%i]/stEvt:softwareAgent'%i].value - data['Timestamp'] = res['Xmp.xmpMM.History[%i]/stEvt:when'%i].value - data['Action'] = res['Xmp.xmpMM.History[%i]/stEvt:action'%i].value - if ('Xmp.xmpMM.History[%i]/stEvt:changed'%i) in res: - data['Info'] = res['Xmp.xmpMM.History[%i]/stEvt:changed'%i].value - #print res['Xmp.xmpMM.History[%i]/stEvt:instanceID'%i].value - result.append( data ) - elif ('Xmp.xmpMM.History[%i]/stEvt:parameters'%i) in res: - data['Action'] = res['Xmp.xmpMM.History[%i]/stEvt:action'%i].value - data['Info'] = res['Xmp.xmpMM.History[%i]/stEvt:parameters'%i].value - #data['Action'] = data['Info'].split(' ')[0] - result.append( data ) - else: - pass - i += 1 - - self._features['History'] = result - return - - class _JpegFile(_UnknownFile): # for '_detect_Trained' cascade_files = [(u'Legs', 'haarcascade_lowerbody.xml'), @@ -2258,6 +2220,45 @@ self._features['Faces'] += data return + def _detect_History(self): + res = self._util_get_DataTags_EXIF() + + #a = [] + #for k in res.keys(): + # if 'history' in k.lower(): + # a.append( k ) + #for item in sorted(a): + # print item + # http://tilloy.net/dev/pyexiv2/api.html#pyexiv2.xmp.XmpTag + #print [getattr(res['Xmp.xmpMM.History'], item) for item in ['key', 'type', 'name', 'title', 'description', 'raw_value', 'value', ]] + result = [] + i = 1 + while (('Xmp.xmpMM.History[%i]' % i) in res): + data = { 'ID': i, + 'Software': u'-', + 'Timestamp': u'-', + 'Action': u'-', + 'Info': u'-', } + if ('Xmp.xmpMM.History[%i]/stEvt:softwareAgent'%i) in res: + data['Software'] = res['Xmp.xmpMM.History[%i]/stEvt:softwareAgent'%i].value + data['Timestamp'] = res['Xmp.xmpMM.History[%i]/stEvt:when'%i].value + data['Action'] = res['Xmp.xmpMM.History[%i]/stEvt:action'%i].value + if ('Xmp.xmpMM.History[%i]/stEvt:changed'%i) in res: + data['Info'] = res['Xmp.xmpMM.History[%i]/stEvt:changed'%i].value + #print res['Xmp.xmpMM.History[%i]/stEvt:instanceID'%i].value + result.append( data ) + elif ('Xmp.xmpMM.History[%i]/stEvt:parameters'%i) in res: + data['Action'] = res['Xmp.xmpMM.History[%i]/stEvt:action'%i].value + data['Info'] = res['Xmp.xmpMM.History[%i]/stEvt:parameters'%i].value + #data['Action'] = data['Info'].split(' ')[0] + result.append( data ) + else: + pass + i += 1 + + self._features['History'] = result + return + def _util_merge_Regions(self, regs, sub=False, overlap=False, close=False): # sub=False, overlap=False, close=False ; level 0 ; similar regions, similar position (default) # sub=True, overlap=False, close=False ; level 1 ; region contained in other, any shape/size @@ -3345,8 +3346,10 @@ # Category:Created_with_OpenOffice.org (pdf) # Category:Created_with_Tux_Paint (pdf) # Category:Created_with_Microsoft_Image_Composite_Editor (jpg) - def _cat_meta_general(self): - result = self._info_filter['Metadata'] + def _cat_meta_and_history_general(self): + results = self._info_filter['Metadata'] +\ + [{'*': item['Software']} for item in self._info_filter['History']] + cats = set() for key, magic, cat in [('Desc', u"Generated automatically by: GNU LilyPond", u'MIDI files created with GNU LilyPond'), ('Software', u"www.inkscape.org", u'Bitmap from Inkscape'), ('Misc', u"org.inkscape.output.svg.inkscape", u'Created with Inkscape'), # 'Output_extension' @@ -3376,12 +3379,13 @@ ('Comment', u"LEAD Technologies Inc.", u'Created with PhotoStitch'), ('Producer', u"Scribus PDF Library", u'Created with Scribus'), ('Producer', u"OpenOffice.org", u'Created with OpenOffice.org'),]: - relevance = len(result) and (key in result[0]) and \ - (magic in result[0][key]) - if relevance: - break + for result in results: + relevance = ((key in result) or ('*' in result)) and \ + (magic in result.get(key, result.get('*'))) + if relevance: + cats.add( cat ) - return (cat, bool(relevance)) + return (list(cats), bool(len(cats))) # Category:Categorized by DrTrigonBot def _addcat_BOT(self): @@ -3654,10 +3658,13 @@ # categorization: use explicit searches for classification (rel = ?) for item in self._funcs['cat']: - (cat, rel) = getattr(self, item)() + (cats, rel) = getattr(self, item)() #print cat, result, len(result) + if not isinstance(cats, list): # because of 'Histroy' and '_cat_meta_and_history_general' + cats = [cats] # which return multiple results... if rel: - self._result_check.append( cat ) + for cat in cats: + self._result_check.append( cat ) self._result_check = list(set(self._result_check)) # categorization: conditional (only if the ones before are present) @@ -4073,6 +4080,11 @@ result = self._info['Streams'] return {'Streams': result} + def _filter_History(self): + # use all, (should be reliable) + result = self._info['History'] + return {'History': result} + # def _filter_Audio(self): # # use all, (should be reliable) # result = self._info['Audio']

1 0

SVN: [11717] trunk/pywikipedia/pywikibot/throttle.py
by drtrigon＠svn.wikimedia.org 07 Jul '13

07 Jul '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11717 Revision: 11717 Author: drtrigon Date: 2013-07-07 09:39:57 +0000 (Sun, 07 Jul 2013) Log Message: ----------- improvement; skip 'get_throttle' and 'put_throttle' in 'simulate=True' mode for speed-up Modified Paths: -------------- trunk/pywikipedia/pywikibot/throttle.py Modified: trunk/pywikipedia/pywikibot/throttle.py =================================================================== --- trunk/pywikipedia/pywikibot/throttle.py 2013-07-06 21:15:48 UTC (rev 11716) +++ trunk/pywikipedia/pywikibot/throttle.py 2013-07-07 09:39:57 UTC (rev 11717) @@ -233,6 +233,11 @@ until the wait expires. """ + if pywikibot.simulate: + pywikibot.output(u'\03{lightyellow}SIMULATION: %s_throttle skipped.\03{default}'%\ + {True: u'put', False: u'get'}[write]) + return + self.lock.acquire() try: wait = self.waittime(write=write or self.write)

1 0

SVN: [11716] branches/rewrite/pywikibot/site.py
by legoktm＠svn.wikimedia.org 07 Jul '13

07 Jul '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11716 Revision: 11716 Author: legoktm Date: 2013-07-06 21:15:48 +0000 (Sat, 06 Jul 2013) Log Message: ----------- Use the deprecate_arg decorator in addition to the debug message Modified Paths: -------------- branches/rewrite/pywikibot/site.py Modified: branches/rewrite/pywikibot/site.py =================================================================== --- branches/rewrite/pywikibot/site.py 2013-07-06 21:05:53 UTC (rev 11715) +++ branches/rewrite/pywikibot/site.py 2013-07-06 21:15:48 UTC (rev 11716) @@ -1862,6 +1862,7 @@ @deprecate_arg("throttle", None) @deprecate_arg("limit", "total") + @deprecate_arg("includeredirects", "filterredir") def allpages(self, start="!", prefix="", namespace=0, filterredir=None, filterlanglinks=None, minsize=None, maxsize=None, protect_type=None, protect_level=None, reverse=False,

1 0

SVN: [11715] branches/rewrite/pywikibot/page.py
by legoktm＠svn.wikimedia.org 07 Jul '13

07 Jul '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11715 Revision: 11715 Author: legoktm Date: 2013-07-06 21:05:53 +0000 (Sat, 06 Jul 2013) Log Message: ----------- Add the ability to parse and read existing qualifiers Modified Paths: -------------- branches/rewrite/pywikibot/page.py Modified: branches/rewrite/pywikibot/page.py =================================================================== --- branches/rewrite/pywikibot/page.py 2013-07-06 05:09:46 UTC (rev 11714) +++ branches/rewrite/pywikibot/page.py 2013-07-06 21:05:53 UTC (rev 11715) @@ -2628,7 +2628,8 @@ """ Claims are standard claims as well as references. """ - def __init__(self, site, pid, snak=None, hash=None, isReference=False): + def __init__(self, site, pid, snak=None, hash=None, isReference=False, + isQualifier=False): """ Defined by the "snak" value, supplemented by site + pid """ @@ -2636,7 +2637,11 @@ self.snak = snak self.hash = hash self.isReference = isReference + self.isQualifier = isQualifier + if self.isQualifier and self.isReference: + raise ValueError(u'Claim cannot be both a qualifier and reference.') self.sources = [] + self.qualifiers = [] self.target = None self.snaktype = 'value' self.on_item = None # The item it's on @@ -2650,8 +2655,11 @@ claim = Claim(site, data['mainsnak']['property']) if 'id' in data: claim.snak = data['id'] + elif 'hash' in data: + claim.isReference = True + claim.hash = data['hash'] else: - claim.isReference = True + claim.isQualifier = True claim.snaktype = data['mainsnak']['snaktype'] if claim.getSnakType() == 'value': if claim.getType() == 'wikibase-item': @@ -2668,6 +2676,10 @@ if 'references' in data: for source in data['references']: claim.sources.append(Claim.referenceFromJSON(site, source)) + if 'qualifiers' in data: + for prop in data['qualifiers']: + for qualifier in data['qualifiers'][prop]: + claim.qualifiers.append(Claim.qualifierFromJSON(site, qualifier)) return claim @staticmethod @@ -2678,11 +2690,20 @@ more handling. """ mainsnak = data['snaks'].values()[0][0] - wrap = {'mainsnak': mainsnak} - c = Claim.fromJSON(site, wrap) - c.hash = data['hash'] - return c + wrap = {'mainsnak': mainsnak, 'hash': data['hash']} + return Claim.fromJSON(site, wrap) + @staticmethod + def qualifierFromJSON(site, data): + """ + Qualifier objects are represented a bit + differently like references, but I'm not + sure if this even requires it's own function. + """ + wrap = {'mainsnak': data} + return Claim.fromJSON(site, wrap) + + def setTarget(self, value): """ Sets the target to the passed value.

1 0

SVN: [11714] trunk/pywikipedia/checkimages.py
by amir＠svn.wikimedia.org 06 Jul '13

06 Jul '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11714 Revision: 11714 Author: amir Date: 2013-07-06 05:09:46 +0000 (Sat, 06 Jul 2013) Log Message: ----------- a little update for Persian Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2013-07-05 19:13:00 UTC (rev 11713) +++ trunk/pywikipedia/checkimages.py 2013-07-06 05:09:46 UTC (rev 11714) @@ -502,7 +502,7 @@ 'ar': 'تصنيف:قوالب حقوق الصور', 'de': 'Kategorie:Vorlage:Lizenz für Bilder', 'en': 'Category:Wikipedia image copyright templates', - 'fa': u'رده:الگوهای حق تکثیر پرونده', + 'fa': u'رده:الگو:حق تکثیر پرونده', 'ga': 'Catagóir:Clibeanna cóipchirt d\'íomhánna', 'it': 'Categoria:Template Licenze copyright', 'ja': 'Category:画像の著作権表示テンプレート',

1 0

SVN: [11713] trunk/pywikipedia/catimages.py
by drtrigon＠svn.wikimedia.org 06 Jul '13

06 Jul '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11713 Revision: 11713 Author: drtrigon Date: 2013-07-05 19:13:00 +0000 (Fri, 05 Jul 2013) Log Message: ----------- bug fix; libmagic (python-magic) has problems with at least 1 jpeg that all other software has not Modified Paths: -------------- trunk/pywikipedia/catimages.py Modified: trunk/pywikipedia/catimages.py =================================================================== --- trunk/pywikipedia/catimages.py 2013-07-05 01:17:44 UTC (rev 11712) +++ trunk/pywikipedia/catimages.py 2013-07-05 19:13:00 UTC (rev 11713) @@ -50,7 +50,7 @@ import re, urllib2, os, locale, sys, datetime, math, shutil, mimetypes, shelve import StringIO, json # fallback: simplejson from subprocess import Popen, PIPE -import Image +import Image, imghdr #import ImageFilter scriptdir = os.path.dirname(sys.argv[0]) @@ -3083,6 +3083,10 @@ m = magic.open(magic.MAGIC_MIME) # or 'magic.MAGIC_NONE' m.load() file_mime = re.split('[/;\s]', m.file(file_name)) + file_imgh = ['image', imghdr.what(file_name)] # alternative MIME ... + if file_imgh[1] and (not (file_imgh == file_mime[:2])): + pywikibot.warning(u'Issue in MIME type detection! Preferring imghdr result %s over libmagic %s!' % (file_imgh, file_mime)) + file_mime = file_imgh + file_mime[2:] mime = mimetypes.guess_all_extensions('%s/%s' % tuple(file_mime[0:2])) if mime and (os.path.splitext(file_name)[1].lower() not in mime): pywikibot.warning(u'File extension does not match MIME type! File extension should be %s.' % mime)

1 0

SVN: [11712] trunk/pywikipedia/checkimages.py
by amir＠svn.wikimedia.org 05 Jul '13

05 Jul '13

http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11712 Revision: 11712 Author: amir Date: 2013-07-05 01:17:44 +0000 (Fri, 05 Jul 2013) Log Message: ----------- a simple update for Persian to make it consistent with regex Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2013-07-03 18:18:11 UTC (rev 11711) +++ trunk/pywikipedia/checkimages.py 2013-07-05 01:17:44 UTC (rev 11712) @@ -142,7 +142,7 @@ 'ar': [u'{{لت', u'{{لا ترخيص'], 'de': [u'{{DÜP', u'{{Düp', u'{{Dateiüberprüfung'], 'en': [u'{{nld', u'{{no license'], - 'fa': [u'{{حق تکثیر تصویر نامعلوم'], + 'fa': [u'{{حق تکثیر تصویر نامعلوم۲'], 'ga': [u'{{Ceadúnas de dhíth', u'{{Ceadúnas de dhíth'], 'hu': [u'{{nincsforrás', u'{{nincslicenc'], 'it': [u'{{unverdata', u'{{unverified'],

1 0

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn July 2013