Pywikibot-commits

pywikibot-commits@lists.wikimedia.org

1 participants
13418 discussions

[Gerrit] lonelypages.py: added i18n TW page - change (pywikibot/i18n)
by jenkins-bot (Code Review) 24 Dec '13

24 Dec '13

jenkins-bot has submitted this change and it was merged. Change subject: lonelypages.py: added i18n TW page ...................................................................... lonelypages.py: added i18n TW page Added TranslateWiki page for lonelypages.py. See Ic832222ec22c90842388ab78c2220af15a567802 for lonelypages.py script. Change-Id: I9b3506d1e801909a2b1d25f6d82063cdca195d26 --- A lonelypages.py 1 file changed, 37 insertions(+), 0 deletions(-) Approvals: Merlijn van Deen: Looks good to me, approved jenkins-bot: Verified diff --git a/lonelypages.py b/lonelypages.py new file mode 100644 index 0000000..e4ab230 --- /dev/null +++ b/lonelypages.py @@ -0,0 +1,37 @@ +# -*- coding: utf-8 -*- +msg = { + 'ar': { + 'lonelypages-comment-add-template': u'بوت: صفحة يتيمة، إضافة قالب', + 'lonelypages-comment-add-disambig-template': u'بوت: إضافة صفحة توضيح', + }, + + 'ca': { + 'lonelypages-comment-add-template': u'Bot:Pàgina orfe, afegint plantilla', + 'lonelypages-comment-add-disambig-template': u'Bot; Afegint una desambiguació', + }, + + 'en': { + 'lonelypages-comment-add-template': u'Robot: Orphan page, add template', + 'lonelypages-comment-add-disambig-template': u'Robot: Adding a disambig page', + }, + + 'it': { + 'lonelypages-comment-add-template': u'[[Project:Bot|Bot]]: Voce orfana, aggiungo template {{O}}', + 'lonelypages-comment-add-disambig-template': u'[[Project:Bot|Bot]]: Aggiungo una disambigua', + }, + + 'ja': { + 'lonelypages-comment-add-template': u'ロボットによる: 孤立したページへのテンプレート貼付け', + 'lonelypages-comment-add-disambig-template': u'ロボットによる: 曖昧さ回避の追加', + }, + + 'zh': { + 'lonelypages-comment-add-template': u'機器人: 本頁的鏈入頁面太少', + 'lonelypages-comment-add-disambig-template': u'機器人: 增加消歧義頁面', + }, + + 'qqq': { + 'lonelypages-comment-add-template': u'Edit summary when adding template to orphan page', + 'lonelypages-comment-add-disambig-template': u'Edit summary when adding lonely page to disambiguation page', + }, +}; -- To view, visit https://gerrit.wikimedia.org/r/103341 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I9b3506d1e801909a2b1d25f6d82063cdca195d26 Gerrit-PatchSet: 3 Gerrit-Project: pywikibot/i18n Gerrit-Branch: master Gerrit-Owner: Gerrit Patch Uploader <gerritpatchuploader(a)gmail.com> Gerrit-Reviewer: Gerrit Patch Uploader <gerritpatchuploader(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] Archivebot: support all languages and timezones. - change (pywikibot/core)
by jenkins-bot (Code Review) 24 Dec '13

24 Dec '13

jenkins-bot has submitted this change and it was merged. Change subject: Archivebot: support all languages and timezones. ...................................................................... Archivebot: support all languages and timezones. Necessary info are retrieved directly from site. No use of locale should be necessary any longer. Patch adapted for core by Merlijn van Deen <valhallasw(a)gmail.com> Change-Id: Iede5165fd36b8e5747db032183094fa11177b037 --- M scripts/archivebot.py 1 file changed, 249 insertions(+), 170 deletions(-) Approvals: Xqt: Looks good to me, approved Merlijn van Deen: Looks good to me, but someone else must approve jenkins-bot: Verified Whym: Looks good to me, but someone else must approve diff --git a/scripts/archivebot.py b/scripts/archivebot.py index fe40a96..968fcd8 100644 --- a/scripts/archivebot.py +++ b/scripts/archivebot.py @@ -36,7 +36,7 @@ algo specifies the maximum age of a thread. Must be in the form old(<delay>) where <delay> specifies the age in hours or days like 24h or 5d. - Default ist old(24h) + Default is old(24h) counter The current value of a counter which could be assigned as variable. Will be actualized by bot. Initial value is 1. maxarchivesize The maximum archive size before incrementing the counter. @@ -72,14 +72,20 @@ # __version__ = '$Id$' # +import pywikibot +from pywikibot import i18n, pagegenerators +import datetime +import time import os import re -import time import locale import traceback -import string -import urllib -import unicodedata + + +ZERO = datetime.timedelta(0) + +Site = pywikibot.getSite() + try: # Get a constructor for the MD5 hash object import hashlib new_hash = hashlib.md5 @@ -87,11 +93,6 @@ import md5 new_hash = md5.md5 -import pywikibot -from pywikibot import i18n, pagegenerators - - -Site = pywikibot.getSite() language = Site.language() @@ -105,9 +106,7 @@ class MissingConfigError(pywikibot.Error): """The config is missing in the header (either it's in one of the threads - or transcluded from another page). - - """ + or transcluded from another page).""" class AlgorithmError(MalformedConfigError): @@ -116,24 +115,20 @@ class ArchiveSecurityError(pywikibot.Error): """Archive is not a subpage of page being archived and key not specified - (or incorrect). - - """ + (or incorrect).""" def str2time(str): """Accepts a string defining a time period: 7d - 7 days 36h - 36 hours - Returns the corresponding time, measured in seconds. - - """ + Returns the corresponding timedelta object.""" if str[-1] == 'd': - return int(str[:-1]) * 24 * 3600 + return datetime.timedelta(days=int(str[:-1])) elif str[-1] == 'h': - return int(str[:-1]) * 3600 + return datetime.timedelta(hours=int(str[:-1])) else: - return int(str) + return datetime.timedelta(seconds=int(str)) def str2size(str): @@ -142,10 +137,8 @@ 150K - 150 kilobytes 2M - 2 megabytes Returns a tuple (size,unit), where size is an integer and unit is - 'B' (bytes) or 'T' (threads). - - """ - if str[-1] in string.digits: # TODO: de-uglify + 'B' (bytes) or 'T' (threads).""" + if str[-1].isdigit(): # TODO: de-uglify return (int(str), 'B') elif str[-1] in ['K', 'k']: return (int(str[:-1]) * 1024, 'B') @@ -155,43 +148,6 @@ return (int(str[:-1]), 'T') else: return (int(str[:-1]) * 1024, 'B') - - -def int2month(num): - """Returns the locale's full name of month 'num' (1-12).""" - if hasattr(locale, 'nl_langinfo'): - return locale.nl_langinfo(locale.MON_1 + num - 1).decode('utf-8') - Months = ['january', 'february', 'march', 'april', 'may_long', 'june', - 'july', 'august', 'september', 'october', 'november', 'december'] - return Site.mediawiki_message(Months[num - 1]) - - -def int2month_short(num): - """Returns the locale's abbreviated name of month 'num' (1-12).""" - if hasattr(locale, 'nl_langinfo'): - #filter out non-alpha characters - return ''.join([c for c in - locale.nl_langinfo( - locale.ABMON_1 + num - 1).decode('utf-8') - if c.isalpha()]) - Months = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', - 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'] - return Site.mediawiki_message(Months[num - 1]) - - -def txt2timestamp(txt, format): - """Attempts to convert the timestamp 'txt' according to given 'format'. - On success, returns the time tuple; on failure, returns None. - - """ -## print txt, format - try: - return time.strptime(txt, format) - except ValueError: - try: - return time.strptime(txt.encode('utf8'), format) - except: - pass def generateTransclusions(Site, template, namespaces=[]): @@ -205,20 +161,213 @@ yield page +class Months(object): + """ + Generation of look-up dictionaries for months, used by Timestripper() and PageArchiver + """ + + def __init__(self, site=None): + if site is None: + self.site = pywikibot.getSite() + else: + self.site = site + + @classmethod + def queryMonths(self): + months_long = ['january', 'february', 'march', 'april', 'may_long', 'june', + 'july', 'august', 'september', 'october', 'november', 'december'] + months_short = ['jan', 'feb', 'mar', 'apr', 'may', 'jun', + 'jul', 'aug', 'sep', 'oct', 'nov', 'dec'] + + #d[1:12] = {'short': 'orig_short', 'long': 'orig_long} + monthNum2origNames = dict((i, {'short': '', 'long': ''}) for i in range(1, 13)) + origNames2monthNum = dict() + + # site.mediawiki_message() does not support preloading multiple messages in one go + qg = pywikibot.data.api.QueryGenerator( + site=self.site, + meta="allmessages", + ammessages='|'.join(months_long + months_short) + ) + + for el in qg: + orig = el["*"] + eng = el["name"] + try: + month_num = months_long.index(eng) + 1 + monthNum2origNames[month_num]['long'] = orig + except ValueError: + month_num = months_short.index(eng) + 1 + monthNum2origNames[month_num]['short'] = orig + + origNames2monthNum[orig] = month_num + + return monthNum2origNames, origNames2monthNum + + @classmethod + def updateMonths(self, site=None): + if site is None: + self.site = pywikibot.getSite() + else: + self.site = site + self.monthsDicts = self.queryMonths() + + +class tzoneUTC(datetime.tzinfo): + """ + Class building a UTC tzinfo object + """ + + def utcoffset(self, dt): + return ZERO + + def tzname(self, dt): + return 'UTC' + + def dst(self, dt): + return ZERO + + def __repr__(self): + return "%s()" % self.__class__.__name__ + + +class tzoneFixedOffset(datetime.tzinfo): + """ + Class building tzinfo objects for fixed-offset time zones + + @offset: a number indicating fixed offset in minutes east from UTC + @name: a string with name of the timezone""" + + def __init__(self, offset, name): + self.__offset = datetime.timedelta(minutes=offset) + self.__name = name + + def utcoffset(self, dt): + return self.__offset + + def tzname(self, dt): + return self.__name + + def dst(self, dt): + return ZERO + + def __repr__(self): + return "%s(%s, %s)" % ( + self.__class__.__name__, + self.__offset.days * 86400 + self.__offset.seconds, + self.__name + ) + + +class TimeStripper(object): + """ + Find timetstamp in page text and returns it as timezone aware datetime object + """ + + def __init__(self): + self.monthNum2origNames, self.origNames2monthNum = Months.monthsDicts + self.site = Months.site + + self.groups = [u'year', u'month', u'hour', u'time', u'day', u'minute', u'tzinfo'] + + timeR = r'(?P<time>(?P<hour>[0-2]\d)[:\.h](?P<minute>[0-5]\d))' + timeznR = r'$(?P<tzinfo>[A-Z]+)$' + yearR = r'(?P<year>(19|20)\d\d)' + monthR = ur'(?P<month>(%s))' % (u'|'.join(self.origNames2monthNum)) + dayR = r'(?P<day>(3[01]|[12]\d|0?[1-9]))' + + self.ptimeR = re.compile(timeR) + self.timeznR = re.compile(timeznR) + self.yearR = re.compile(yearR) + self.pmonthR = re.compile(monthR, re.U) + self.pdayR = re.compile(dayR) + + #order is important to avoid mismatch when searching + self.patterns = [ + self.ptimeR, + self.timeznR, + self.yearR, + self.pmonthR, + self.pdayR, + ] + + def findmarker(self, text, base=u'@@', delta='@'): + # find a string which is not part of text + while base in text: + base += delta + return base + + def last_match_and_replace(self, txt, pat): + """ + Take the rightmost match, to prevent spurious earlier matches, and replace with marker + """ + m = None + for m in pat.finditer(txt): + pass + + if m: + marker = self.findmarker(txt) + txt = pat.sub(marker, txt) + return (txt, m.groupdict()) + else: + return (txt, None) + + def timestripper(self, line): + """ + Find timestamp in line and convert it to time zone aware datetime + """ + _line = line + #match date fields + dateDict = dict() + for pat in self.patterns: + line, matchDict = self.last_match_and_replace(line, pat) + if matchDict: + dateDict.update(matchDict) + + #all fields matched -> date valid + if all(g in dateDict for g in self.groups): + #remove 'time' key, now splitted in hour/minute and not needed by datetime + del dateDict['time'] + + #replace month name in original language with month number + try: + dateDict['month'] = self.origNames2monthNum[dateDict['month']] + except KeyError: + pywikibot.output(u'incorrect month name in page') + + #convert to integers + for k, v in dateDict.items(): + try: + dateDict[k] = int(v) + except ValueError: + pass + + #find timezone + dateDict['tzinfo'] = tzoneFixedOffset(self.site.siteinfo['timeoffset'], + self.site.siteinfo['timezone']) + + timestamp = datetime.datetime(**dateDict) + + else: + timestamp = None + + return timestamp + + class DiscussionThread(object): - """An object representing a discussion thread on a page, that is something - of the form: + """An object representing a discussion thread on a page, that is something of the form: == Title of thread == Thread content here. ~~~~ :Reply, etc. ~~~~ - """ - def __init__(self, title): + def __init__(self, title, now): self.title = title + self.now = now self.content = "" + self.ts = TimeStripper() self.timestamp = None def __repr__(self): @@ -228,86 +377,16 @@ def feedLine(self, line): if not self.content and not line: return - self.content += line + '\n' - #Update timestamp -# nnwiki: -# 19:42, 25 mars 2008 (CET) -# enwiki -# 16:36, 30 March 2008 (UTC) -# huwiki -# 2007. december 8., 13:42 (CET) - TM = re.search(r'(\d\d):(\d\d), (\d\d?) (\S+) (\d\d\d\d) $.*?$', line) - if not TM: - TM = re.search(r'(\d\d):(\d\d), (\S+) (\d\d?), (\d\d\d\d) $.*?$', - line) - if not TM: - TM = re.search(r'(\d{4})\. (\S+) (\d\d?)\., (\d\d:\d\d) $.*?$', - line) -# 18. apr 2006 kl.18:39 (UTC) -# 4. nov 2006 kl. 20:46 (CET) - if not TM: - TM = re.search(r'(\d\d?)\. (\S+) (\d\d\d\d) kl\.\W*(\d\d):(\d\d) $.*?$', - line) -#3. joulukuuta 2008 kello 16.26 (EET) - if not TM: - TM = re.search(r'(\d\d?)\. (\S+) (\d\d\d\d) kello \W*(\d\d).(\d\d) $.*?$', - line) - if not TM: -# 14:23, 12. Jan. 2009 (UTC) - pat = re.compile(r'(\d\d):(\d\d), (\d\d?)\. (\S+)\.? (\d\d\d\d) $(?:UTC|CES?T)$') - TM = pat.search(line) -# ro.wiki: 4 august 2012 13:01 (EEST) - if not TM: - TM = re.search(r'(\d\d?) (\S+) (\d\d\d\d) (\d\d):(\d\d) $.*?$', - line) -# Japanese: 2012年8月4日 (日) 13:01 (UTC) - if not TM: - TM = re.search(re.compile(u'(\d\d\d\d)年(\d\d?)月(\d\d?)日 $.$ (\d\d):(\d\d) $.*?$'), - line) - if TM: - # Strip away all diacritics in the Mn ('Mark, non-spacing') category - # NFD decomposition splits combined characters (e.g. 'ä", - # LATIN SMALL LETTER A WITH DIAERESIS) into two entities: - # LATIN SMALL LETTER A and COMBINING DIAERESIS. The latter falls - # in the Mn category and is filtered out, resuling in 'a'. - _TM = ''.join(c for c in unicodedata.normalize('NFD', TM.group(0)) - if unicodedata.category(c) != 'Mn') - TIME = txt2timestamp(_TM, "%d. %b %Y kl. %H:%M (%Z)") - if not TIME: - TIME = txt2timestamp(_TM, "%Y. %B %d., %H:%M (%Z)") - if not TIME: - TIME = txt2timestamp(_TM, "%d. %b %Y kl.%H:%M (%Z)") - if not TIME: - TIME = txt2timestamp(re.sub(' *$[^ ]+$ *', '', _TM), - "%H:%M, %d %B %Y") - if not TIME: - TIME = txt2timestamp(_TM, "%H:%M, %d %b %Y (%Z)") - if not TIME: - TIME = txt2timestamp(re.sub(' *$[^ ]+$ *', '', _TM), - "%H:%M, %d %b %Y") - if not TIME: - TIME = txt2timestamp(_TM, "%H:%M, %b %d %Y (%Z)") - if not TIME: - TIME = txt2timestamp(_TM, "%H:%M, %B %d %Y (%Z)") - if not TIME: - TIME = txt2timestamp(_TM, "%H:%M, %b %d, %Y (%Z)") - if not TIME: - TIME = txt2timestamp(_TM, "%H:%M, %B %d, %Y (%Z)") - if not TIME: - TIME = txt2timestamp(_TM, "%d. %Bta %Y kello %H.%M (%Z)") - if not TIME: - TIME = txt2timestamp(_TM, "%d %B %Y %H:%M (%Z)") - if not TIME: - TIME = txt2timestamp(_TM, "%Y年%B%d日 (%a) %H:%M (%Z)") - if not TIME: - TIME = txt2timestamp(re.sub(' *$[^ ]+$ *', '', _TM), - "%H:%M, %d. %b. %Y") - if TIME: - self.timestamp = max(self.timestamp, time.mktime(TIME)) -## pywikibot.output(u'Time to be parsed: %s' % TM.group(0)) -## pywikibot.output(u'Parsed time: %s' % TIME) -## pywikibot.output(u'Newest timestamp in thread: %s' % TIME) + self.content += line + '\n' + + timestamp = self.ts.timestripper(line) + + if not self.timestamp: # first time + self.timestamp = timestamp + + if timestamp: + self.timestamp = max(self.timestamp, timestamp) def size(self): return len(self.title) + len(self.content) + 12 @@ -324,16 +403,14 @@ #TODO: handle this: #return 'unsigned' maxage = str2time(reT.group(1)) - if self.timestamp + maxage < time.time(): + if self.now - self.timestamp > maxage: return message('archivebot-older-than') + ' ' + reT.group(1) return '' class DiscussionPage(pywikibot.Page): """A class that represents a single discussion page as well as an archive - page. Feed threads to it and run an update() afterwards. - - """ + page. Feed threads to it and run an update() afterwards.""" def __init__(self, title, archiver, vars=None): pywikibot.Page.__init__(self, Site, title) @@ -341,6 +418,8 @@ self.full = False self.archiver = archiver self.vars = vars + self.now = datetime.datetime.utcnow().replace(tzinfo=tzoneUTC()) + try: self.loadPage() except pywikibot.NoPage: @@ -364,7 +443,7 @@ found = True # Reading threads now if curThread: self.threads.append(curThread) - curThread = DiscussionThread(threadHeader.group(1)) + curThread = DiscussionThread(threadHeader.group(1), self.now) else: if found: curThread.feedLine(line) @@ -425,6 +504,7 @@ } self.archives = {} self.archivedThreads = 0 + self.monthNum2origNames, self.origNames2monthNum = Months.monthsDicts def get(self, attr, default=''): return self.attributes.get(attr, [default])[0] @@ -439,10 +519,10 @@ and a != 'maxage'] def attr2text(self): - return '{{%s\n%s\n}}' % (self.tpl, - '\n'.join(['|%s = %s ' - % (a, self.get(a)) - for a in self.saveables()])) + return '{{%s\n%s\n}}' \ + % (self.tpl, + '\n'.join(['|%s = %s' % (a, self.get(a)) + for a in self.saveables()])) def key_ok(self): s = new_hash() @@ -474,10 +554,9 @@ if not archive: return if not self.force \ - and not self.Page.title() + '/' == archive[ - :len(self.Page.title()) + 1] \ + and not self.Page.title() + '/' == archive[:len(self.Page.title()) + 1] \ and not self.key_ok(): - raise ArchiveSecurityError + raise ArchiveSecurityError("Archive page %r does not start with page title (%s)!" % (archive, self.Page.title())) if not archive in self.archives: self.archives[archive] = DiscussionPage(archive, self, vars) return self.archives[archive].feedThread(thread, maxArchiveSize) @@ -487,7 +566,6 @@ archCounter = int(self.get('counter', '1')) oldthreads = self.Page.threads self.Page.threads = [] - T = time.mktime(time.gmtime()) whys = [] pywikibot.output(u'Processing %d threads' % len(oldthreads)) for t in oldthreads: @@ -500,14 +578,13 @@ why = t.shouldBeArchived(self) if why: archive = self.get('archive') - TStuple = time.gmtime(t.timestamp) vars = { 'counter': archCounter, - 'year': TStuple[0], - 'month': TStuple[1], - 'monthname': int2month(TStuple[1]), - 'monthnameshort': int2month_short(TStuple[1]), - 'week': int(time.strftime('%W', TStuple)), + 'year': t.timestamp.year, + 'month': t.timestamp.month, + 'monthname': self.monthNum2origNames[t.timestamp.month]['long'], + 'monthnameshort': self.monthNum2origNames[t.timestamp.month]['short'], + 'week': int(time.strftime('%W', t.timestamp.timetuple())), } archive = pywikibot.Page(Site, archive % vars).title() if self.feedArchive(archive, t, maxArchSize, vars): @@ -539,12 +616,12 @@ self.commentParams) self.archives[a].update(comment) - #Save the page itself - rx = re.compile('{{%s\n.*?\n}}' % self.tpl, re.DOTALL) + # Save the page itself + rx = re.compile('{{' + self.tpl + '\n.*?\n}}', re.DOTALL) self.Page.header = rx.sub(self.attr2text(), self.Page.header) self.commentParams['count'] = self.archivedThreads - self.commentParams['archives'] = ', '.join( - ['[[%s]]' % a.title() for a in self.archives.values()]) + self.commentParams['archives'] \ + = ', '.join(['[[' + a.title() + ']]' for a in self.archives.values()]) if not self.commentParams['archives']: self.commentParams['archives'] = '/dev/null' self.commentParams['why'] = ', '.join(whys) @@ -617,6 +694,9 @@ pywikibot.showHelp('archivebot') return + #query site for original months name and create convenience look-up dictionaries + Months.updateMonths(site=Site) + for a in args[1:]: pagelist = [] a = a.decode('utf8') @@ -645,7 +725,6 @@ except: pywikibot.output(u'Error occured while processing page %s' % pg) traceback.print_exc() - if __name__ == '__main__': try: -- To view, visit https://gerrit.wikimedia.org/r/103324 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Iede5165fd36b8e5747db032183094fa11177b037 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Mpaa <mpaa.wiki(a)gmail.com> Gerrit-Reviewer: Whym <whym(a)whym.org> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] bugfix; UnicodeEncodeError: 'ascii' codec can't encode chara... - change (pywikibot/compat)
by jenkins-bot (Code Review) 23 Dec '13

23 Dec '13

jenkins-bot has submitted this change and it was merged. Change subject: bugfix; UnicodeEncodeError: 'ascii' codec can't encode character u'\xba' in position 62: ordinal not in range(128) ...................................................................... bugfix; UnicodeEncodeError: 'ascii' codec can't encode character u'\xba' in position 62: ordinal not in range(128) Change-Id: I9480323afdd482e6388520354af8a8784253c2cb --- M sum_disc.py 1 file changed, 2 insertions(+), 1 deletion(-) Approvals: DrTrigon: Verified Merlijn van Deen: Looks good to me, approved jenkins-bot: Verified diff --git a/sum_disc.py b/sum_disc.py index bff8de6..1d865fc 100644 --- a/sum_disc.py +++ b/sum_disc.py @@ -524,7 +524,8 @@ self._param['ignorepage_list'].append(self._userPage.title()) self._datfilename = pywikibot.config.datafilepath( self._bot_config['data_path'], 'sum_disc-%s-%s-%s.dat' - % (self.site.family.name, self.site.lang, self._user.name())) + % (self.site.family.name, self.site.lang, + self._user.name())).encode('utf-8') # substitute variables for use in user defined parameters/options param_vars = {'username': self._user.name(), -- To view, visit https://gerrit.wikimedia.org/r/103348 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I9480323afdd482e6388520354af8a8784253c2cb Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/compat Gerrit-Branch: master Gerrit-Owner: DrTrigon <dr.trigon(a)surfeu.ch> Gerrit-Reviewer: DrTrigon <dr.trigon(a)surfeu.ch> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] Rename 'pg' back to 'pagegenerators' - change (pywikibot/compat)
by jenkins-bot (Code Review) 23 Dec '13

23 Dec '13

jenkins-bot has submitted this change and it was merged. Change subject: Rename 'pg' back to 'pagegenerators' ...................................................................... Rename 'pg' back to 'pagegenerators' Change-Id: Ia80e09a8811f31ad927ba802c6e1515ddd030359 --- M template.py 1 file changed, 10 insertions(+), 8 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/template.py b/template.py index ac1c2a1..f75797e 100644 --- a/template.py +++ b/template.py @@ -108,9 +108,9 @@ # import re -import wikipedia as pywikibot +import pywikibot from pywikibot import i18n -import pagegenerators as pg +import pagegenerators import xmlreader import replace import catlib @@ -291,7 +291,7 @@ editSummary = '' addedCat = '' acceptAll = False - genFactory = pg.GeneratorFactory() + genFactory = pagegenerators.GeneratorFactory() # If xmlfilename is None, references will be loaded from the live wiki. xmlfilename = None user = None @@ -356,12 +356,14 @@ else: gen = genFactory.getCombinedGenerator() if not gen: - gens = [pg.ReferringPageGenerator(t, onlyTemplateInclusion=True) - for t in oldTemplates] - gen = pg.CombinedPageGenerator(gens) - gen = pg.DuplicateFilterPageGenerator(gen) + gens = [ + pagegenerators.ReferringPageGenerator(t, onlyTemplateInclusion=True) + for t in oldTemplates + ] + gen = pagegenerators.CombinedPageGenerator(gens) + gen = pagegenerators.DuplicateFilterPageGenerator(gen) - preloadingGen = pg.PreloadingGenerator(gen) + preloadingGen = pagegenerators.PreloadingGenerator(gen) bot = TemplateRobot(preloadingGen, templates, subst, remove, editSummary, acceptAll, addedCat) -- To view, visit https://gerrit.wikimedia.org/r/103254 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ia80e09a8811f31ad927ba802c6e1515ddd030359 Gerrit-PatchSet: 2 Gerrit-Project: pywikibot/compat Gerrit-Branch: master Gerrit-Owner: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] [SYNC] Synchronize with core, code improvements - change (pywikibot/compat)
by jenkins-bot (Code Review) 23 Dec '13

23 Dec '13

jenkins-bot has submitted this change and it was merged. Change subject: [SYNC] Synchronize with core, code improvements ...................................................................... [SYNC] Synchronize with core, code improvements - remove obsolete config, sys and string import - remove unused variables Change-Id: I88a29b5e3654eb70df412c3f1d65182ccd02811b --- M template.py 1 file changed, 14 insertions(+), 12 deletions(-) Approvals: Xqt: Looks good to me, approved Huji: Checked; Looks good to me, but someone else must approve jenkins-bot: Verified diff --git a/template.py b/template.py index 33c7ef0..ac1c2a1 100644 --- a/template.py +++ b/template.py @@ -1,3 +1,4 @@ +#!/usr/bin/python # -*- coding: utf-8 -*- """ Very simple script to replace a template with another one, @@ -105,15 +106,14 @@ # __version__ = '$Id$' # + import re -import sys -import string import wikipedia as pywikibot from pywikibot import i18n -import config -import catlib import pagegenerators as pg +import xmlreader import replace +import catlib def UserEditFilterGenerator(generator, username, timestamp=None, skip=False): @@ -159,13 +159,11 @@ * xmlfilename - The dump's path, either absolute or relative """ - self.templates = templates self.xmlfilename = xmlfilename def __iter__(self): """Yield page objects until the entire XML dump has been read.""" - import xmlreader mysite = pywikibot.getSite() dump = xmlreader.XmlDump(self.xmlfilename) # regular expression to find the original template. @@ -176,11 +174,14 @@ for template in self.templates: templatePattern = template.title(withNamespace=False) if not pywikibot.getSite().nocapitalize: - templatePattern = '[' + templatePattern[0].upper() + templatePattern[0].lower() + ']' + templatePattern[1:] + templatePattern = '[%s%s]%s' % (templatePattern[0].upper(), + templatePattern[0].lower(), + templatePattern[1:]) templatePattern = re.sub(' ', '[_ ]', templatePattern) templatePatterns.append(templatePattern) - templateRegex = re.compile(r'\{\{ *([mM][sS][gG]:)?(?:%s) *(?P<parameters>\|[^}]+|) *}}' % '|'.join(templatePatterns)) - + templateRegex = re.compile( + r'\{\{ *([mM][sS][gG]:)?(?:%s) *(?P<parameters>\|[^}]+|) *}}' + % '|'.join(templatePatterns)) for entry in dump.parse(): if templateRegex.search(entry.text): page = pywikibot.Page(mysite, entry.title) @@ -219,7 +220,7 @@ self.addedCat = catlib.Category( site, u'%s:%s' % (site.namespace(14), self.addedCat)) - comma = self.summary = site.mediawiki_message('comma-separator') + comma = site.mediawiki_message('comma-separator') # get edit summary message if it's empty if not self.editSummary: @@ -326,7 +327,8 @@ else: if not genFactory.handleArg(arg): templateNames.append( - pywikibot.Page(pywikibot.getSite(), arg, defaultNamespace=10 + pywikibot.Page(pywikibot.getSite(), arg, + defaultNamespace=10 ).title(withNamespace=False)) if subst ^ remove: @@ -342,7 +344,6 @@ return oldTemplates = [] - ns = pywikibot.getSite().template_namespace() for templateName in templates.keys(): oldTemplate = pywikibot.Page(pywikibot.getSite(), templateName, defaultNamespace=10) @@ -361,6 +362,7 @@ gen = pg.DuplicateFilterPageGenerator(gen) preloadingGen = pg.PreloadingGenerator(gen) + bot = TemplateRobot(preloadingGen, templates, subst, remove, editSummary, acceptAll, addedCat) bot.run() -- To view, visit https://gerrit.wikimedia.org/r/102127 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I88a29b5e3654eb70df412c3f1d65182ccd02811b Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/compat Gerrit-Branch: master Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Huji <huji.huji(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Siebrand <siebrand(a)wikimedia.org> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] [PEP8] changes, code improvements - change (pywikibot/compat)
by Alex S.H. Lin (Code Review) 23 Dec '13

23 Dec '13

Alex S.H. Lin has submitted this change and it was merged. Change subject: [PEP8] changes, code improvements ...................................................................... [PEP8] changes, code improvements Change-Id: I6c7030e61030f4f5450603becc09eedf0b84428e --- M deledpimage.py 1 file changed, 67 insertions(+), 58 deletions(-) Approvals: Alex S.H. Lin: Verified; Looks good to me, approved diff --git a/deledpimage.py b/deledpimage.py index a78d364..ae727ad 100644 --- a/deledpimage.py +++ b/deledpimage.py @@ -9,24 +9,23 @@ * [[:Image:logo.jpg]] pass * Image:logo.jpg in gallery --> [[:Image:logo.jpg]] in gallery end * logo.jpg(like used in template) --> hide(used ) + +commond: +python deledpimage.py + """ # -# (C) Shizhao, 2008 +# (c) Shizhao, 2008 +# (c) pywikibot team, 2009-2013 # # Distributed under the terms of the MIT license. # __version__ = '$Id$' # -""" -commond: -python deledpimage.py - -删除显示在用户页上的合理使用图像 - -""" -import re, time -import wikipedia as pywikibot +import re +import time +import pywikibot import catlib site = pywikibot.getSite() @@ -35,90 +34,100 @@ 'ar': u'تصنيف:وسوم حقوق نسخ الصور غير الحرة', 'en': u'Category:Non-free image copyright tags', 'zh': u'Category:合理使用图像模板', - } +} content = { 'ar': u'هذه الصورة غير الحرة غير مستخدمة في نطاق المقالات، انظر [[Wikipedia:Non-free content#Policy]]', 'en': u'This Non-free image NOT used in non-article namespaces, see[[Wikipedia:Non-free content#Policy]]', 'zh': u'不是使用在条目中的非自由版权图像，根据[[Wikipedia:合理使用]]，不能在非条目名字空间展示：\n', - } +} msg = { 'ar': u'روبوت: إصلاح استخدام صورة EDP: [[%s]]', 'en': u'Robot: Fix EDP image use: [[%s]]', 'zh': u'Bot修正EDP图像用法：[[%s]]', - } +} -lcontent=pywikibot.translate(site, content) -category=pywikibot.translate(site, cat) -putmsg=pywikibot.translate(site, msg) +lcontent = pywikibot.translate(site, content) +category = pywikibot.translate(site, cat) +putmsg = pywikibot.translate(site, msg) -#from non-free copyright tag category get all EDPtemplate -templatecat=catlib.Category(site, category) +# from non-free copyright tag category get all EDPtemplate +templatecat = catlib.Category(site, category) templatelist = templatecat.articlesList() -#from References of EDP template get all non-free images +# from References of EDP template get all non-free images for tempalte in templatelist: images = [page for page in tempalte.getReferences() if page.isImage()] - for image in images : - imagetitle=image.title() - imagepage=pywikibot.ImagePage(site,imagetitle) + for image in images: + imagetitle = image.title() + imagepage = pywikibot.ImagePage(site, imagetitle) -#from imagepage get all usingPages of non-articles - pimages=[puseimage for puseimage in imagepage.usingPages() if puseimage.namespace()<>0] + # from imagepage get all usingPages of non-articles + pimages = [puseimage for puseimage in imagepage.usingPages() + if puseimage.namespace() != 0] for pimage in pimages: - ns=pimage.namespace() - pimagetitle=pimage.title() + ns = pimage.namespace() + pimagetitle = pimage.title() c = u'\nfond an used the image [[%s]] in [[%s]]: ' \ % (imagetitle, pimagetitle) - text=pimage.get() + text = pimage.get() try: - re.search('',text,re.I).group(0) + re.search('', + text, re.I).group(0) except: try: - # imagetext=re.search('\[\['+imagetitle+'(.*?)\]\]',text,re.I).group(0) if imagetitle not in text: - -# Not [[Image:]] namespace + # Not [[Image:]] namespace if imagetitle[6:] in text: - imagetext = re.search(imagetitle[6:]+'(.*?)(|)',text,re.I).group(0) - text = re.sub(imagetitle[6:]+'(.*?)(|)', '', text, re.I) - pywikibot.output(c+u'remove!!!\nSleep 10 s......') + imagetext = re.search(imagetitle[6:] + '(.*?)(|)', + text, re.I).group(0) + text = re.sub(imagetitle[6:] + '(.*?)(|)', + '', + text, re.I) + pywikibot.output(c + u'remove!!!\nSleeping 10 s...') pimage.put(text, putmsg % imagetitle) time.sleep(10) -#used [[Image:wiki.png]] image + #used [[Image:wiki.png]] image else: - if '[['+imagetitle in text: + if '[[' + imagetitle in text: -#Image in userpage, imagepage,and all talkpage , [[Image:wiki.png]] --> [[:Image:wiki.png]] - if ns==1 or ns==6 or ns==2 or ns==3 or ns==5 or ns==7 or ns==9 or ns==11 or ns==13 or ns==15 or ns==17 or ns==101: + #Image in userpage, imagepage, and all talkpage + #[[Image:wiki.png]] --> [[:Image:wiki.png]] - text = re.sub('\[\['+imagetitle+'(.*?)\]\]', ''+'[['+':'+imagetitle+']]',text, re.I) - pywikibot.output(c+u'FIX!\nSleep 10 s......') - pimage.put(text, putmsg % imagetitle) - time.sleep(10) + if ns in (1, 2, 3, 5, 6, 7, 9, 11, 13, 15, 17, 101): + text = re.sub('\[\[' + imagetitle + '(.*?)\]\]', + '' + + '[[' + ':' + imagetitle + ']]', + text, re.I) + pywikibot.output(c + u'FIX!\nSleeping 10 s...') + pimage.put(text, putmsg % imagetitle) + time.sleep(10) -#Image in template, categorypage, remove - elif ns==10 or ns==14: - text = re.sub('\[\['+imagetitle+'(.*?)(|)\]\]', '',text, re.I) - pywikibot.output(c+u'Remove!!!\nSleep 10 s......') - pimage.put(text, putmsg % imagetitle) - time.sleep(10) -# elif '[[:'+imagetitle in text: -# pywikibot.output(c+u'EDP is OK!') - -#Image in <gallery></gallery> + #Image in template, categorypage, remove + elif ns in (10, 14): + text = re.sub( + '\[\[' + imagetitle + '(.*?)(|)\]\]', + '', + text, re.I) + pywikibot.output( + c + u'Remove!!!\nSleeping 10 s...') + pimage.put(text, putmsg % imagetitle) + time.sleep(10) + #Image in <gallery></gallery> else: -# try: -# imagetext=re.search(imagetitle+'(.*?)\n',text,re.I).group(0) - text = re.sub(imagetitle+'(.*?)', '',text, re.I) - text=re.sub('</gallery>\n', '</gallery>\n'+'\n'+'[[:'+imagetitle+']]\n',text, re.I) - pywikibot.output(c+u'FIX <gallery>!\nSleep 10 s......') + text = re.sub(imagetitle + '(.*?)', '', text, re.I) + text = re.sub('</gallery>\n', + '</gallery>\n' + '\n' + + '[[:' + imagetitle + ']]\n', + text, re.I) + pywikibot.output( + c + u'FIX <gallery>!\nSleeping 10 s...') pimage.put(text, putmsg % imagetitle) time.sleep(10) except: print 'Error' pass - -- To view, visit https://gerrit.wikimedia.org/r/103255 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I6c7030e61030f4f5450603becc09eedf0b84428e Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/compat Gerrit-Branch: master Gerrit-Owner: Xqt <info(a)gno.de> Gerrit-Reviewer: Alex S.H. Lin <alexsh(a)mail2000.com.tw> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Meno25 <meno25mail(a)gmail.com> Gerrit-Reviewer: Siebrand <siebrand(a)wikimedia.org> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] [PEP8] editarticle.py - change (pywikibot/compat)
by jenkins-bot (Code Review) 23 Dec '13

23 Dec '13

jenkins-bot has submitted this change and it was merged. Change subject: [PEP8] editarticle.py ...................................................................... [PEP8] editarticle.py Change-Id: I5331abeb7848352a7554cc4766415a25d25fce2b --- M editarticle.py 1 file changed, 13 insertions(+), 12 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/editarticle.py b/editarticle.py index da73797..ab6ce12 100644 --- a/editarticle.py +++ b/editarticle.py @@ -37,7 +37,7 @@ def __init__(self): pass - def command(self, tempFilename, text, jumpIndex = None): + def command(self, tempFilename, text, jumpIndex=None): command = config.editor if jumpIndex: # Some editors make it possible to mark occurences of substrings, @@ -53,25 +53,25 @@ if config.editor.startswith('kate'): command += " -l %i -c %i" % (line + 1, column + 1) elif config.editor.startswith('gedit'): - command += " +%i" % (line + 1) # seems not to support columns + command += " +%i" % (line + 1) # seems not to support columns elif config.editor.startswith('emacs'): - command += " +%i" % (line + 1) # seems not to support columns + command += " +%i" % (line + 1) # seems not to support columns elif config.editor.startswith('jedit'): - command += " +line:%i" % (line + 1) # seems not to support columns + command += " +line:%i" % (line + 1) # seems not to support columns elif config.editor.startswith('vim'): - command += " +%i" % (line + 1) # seems not to support columns + command += " +%i" % (line + 1) # seems not to support columns elif config.editor.startswith('nano'): command += " +%i,%i" % (line + 1, column + 1) # Windows editors elif config.editor.lower().endswith('notepad++.exe'): - command += " -n%i" % (line + 1) # seems not to support columns + command += " -n%i" % (line + 1) # seems not to support columns command += ' %s' % tempFilename #print command return command def convertLinebreaks(self, text): - if sys.platform=='win32': + if sys.platform == 'win32': return text.replace('\r\n', '\n') # TODO: Mac OS handling return text @@ -79,12 +79,12 @@ def restoreLinebreaks(self, text): if text is None: return None - if sys.platform=='win32': + if sys.platform == 'win32': return text.replace('\n', '\r\n') # TODO: Mac OS handling return text - def edit(self, text, jumpIndex = None, highlight = None): + def edit(self, text, jumpIndex=None, highlight=None): """ Calls the editor and thus allows the user to change the text. Returns the modified text. Halts the thread's operation until the editor @@ -113,13 +113,14 @@ return None else: newcontent = open(tempFilename).read().decode( - config.editor_encoding) + config.editor_encoding) os.unlink(tempFilename) return self.restoreLinebreaks(newcontent) else: return self.restoreLinebreaks( pywikibot.ui.editText(text, jumpIndex=jumpIndex, highlight=highlight)) + class ArticleEditor: # join lines if line starts with this ones @@ -169,7 +170,7 @@ def run(self): try: - old = self.page.get(get_redirect = self.options.edit_redirect) + old = self.page.get(get_redirect=self.options.edit_redirect) except pywikibot.NoPage: old = "" textEditor = TextEditor() @@ -187,6 +188,7 @@ else: pywikibot.output(u"Nothing changed") + def main(*args): app = ArticleEditor(*args) app.run() @@ -196,4 +198,3 @@ main() finally: pywikibot.stopme() - -- To view, visit https://gerrit.wikimedia.org/r/103329 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I5331abeb7848352a7554cc4766415a25d25fce2b Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/compat Gerrit-Branch: master Gerrit-Owner: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] [PEP8] changes - change (pywikibot/compat)
by jenkins-bot (Code Review) 23 Dec '13

23 Dec '13

jenkins-bot has submitted this change and it was merged. Change subject: [PEP8] changes ...................................................................... [PEP8] changes Change-Id: Icf144501fa5939ccb12ae7278929175551573248 --- M extract_wikilinks.py M fixing_redirects.py M followlive.py M get.py 4 files changed, 23 insertions(+), 19 deletions(-) Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified diff --git a/extract_wikilinks.py b/extract_wikilinks.py index 6f5e7d1..601cea2 100644 --- a/extract_wikilinks.py +++ b/extract_wikilinks.py @@ -19,9 +19,10 @@ # # Distributed under the terms of the MIT license. # -__version__='$Id$' +__version__ = '$Id$' # -import sys,re +import sys +import re import codecs import wikipedia as pywikibot # This bot does not contact the Wiki, so no need to get it on the list @@ -38,7 +39,7 @@ elif arg.startswith("-bare"): complete = False elif fn: - print "Ignoring argument %s"%arg + print "Ignoring argument %s" % arg else: fn = arg @@ -47,14 +48,14 @@ sys.exit(1) mysite = pywikibot.getSite() -f=open(fn,'r') -text=f.read() +f = open(fn, 'r') +text = f.read() f.close() for hit in R.findall(text): if complete: list.append(mysite.linkto(hit)) else: - list.append("[[%s]]"%hit) + list.append("[[%s]]" % hit) if sorted: list.sort() for page in list: diff --git a/fixing_redirects.py b/fixing_redirects.py index 49627dd..d35badf 100644 --- a/fixing_redirects.py +++ b/fixing_redirects.py @@ -20,9 +20,10 @@ # # Distributed under the terms of the MIT license. # -__version__='$Id$' +__version__ = '$Id$' # -import re, sys +import re +import sys import wikipedia as pywikibot import pagegenerators from pywikibot import i18n @@ -72,7 +73,7 @@ curpos = 0 # This loop will run until we have finished the current page while True: - m = linkR.search(text, pos = curpos) + m = linkR.search(text, pos=curpos) if not m: break # Make sure that next time around we will not find this same hit. @@ -102,7 +103,7 @@ if not link_text: # or like this: [[page_title]]trailing_chars link_text = page_title - if m.group('section') == None: + if m.group('section') is None: section = '' else: section = m.group('section') @@ -117,16 +118,16 @@ replaceit = choice in "rR" # remove preleading ":" - if link_text[0]==':': + if link_text[0] == ':': link_text = link_text[1:] if link_text[0].isupper(): new_page_title = targetPage.title() else: new_page_title = targetPage.title()[0].lower() + \ - targetPage.title()[1:] + targetPage.title()[1:] # remove preleading ":" - if new_page_title[0]==':': + if new_page_title[0] == ':': new_page_title = new_page_title[1:] if replaceit and trailing_chars: @@ -149,6 +150,7 @@ pageCache = [] + def workon(page): mysite = pywikibot.getSite() try: @@ -163,7 +165,7 @@ % page.title()) links = page.linkedPages() if len(links): - pywikibot.getall(mysite,links) + pywikibot.getall(mysite, links) else: pywikibot.output('Nothing left to do.') return diff --git a/followlive.py b/followlive.py index a8987a2..ba7a822 100644 --- a/followlive.py +++ b/followlive.py @@ -13,10 +13,11 @@ # # Distributed under the terms of the MIT license. # -__version__='$Id$' +__version__ = '$Id$' import sys -import datetime, time +import datetime +import time import traceback import wikipedia as pywikibot import editarticle diff --git a/get.py b/get.py index 82343c6..4b97cdf 100644 --- a/get.py +++ b/get.py @@ -13,9 +13,10 @@ # # Distributed under the terms of the MIT license. -__version__='$Id$' +__version__ = '$Id$' import wikipedia as pywikibot + def main(): singlePageTitleParts = [] @@ -26,11 +27,10 @@ page = pywikibot.Page(pywikibot.getSite(), pageTitle) # TODO: catch exceptions - pywikibot.output(page.get(), toStdout = True) + pywikibot.output(page.get(), toStdout=True) if __name__ == "__main__": try: main() finally: pywikibot.stopme() - -- To view, visit https://gerrit.wikimedia.org/r/103333 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Icf144501fa5939ccb12ae7278929175551573248 Gerrit-PatchSet: 1 Gerrit-Project: pywikibot/compat Gerrit-Branch: master Gerrit-Owner: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] Added script to upload Freebase identifiers - change (pywikibot/core)
by jenkins-bot (Code Review) 23 Dec '13

23 Dec '13

jenkins-bot has submitted this change and it was merged. Change subject: Added script to upload Freebase identifiers ...................................................................... Added script to upload Freebase identifiers Change-Id: I9d1f459d58da0eecd1631e8b863129d9d1aa3a85 --- A scripts/freebasemappingupload.py 1 file changed, 103 insertions(+), 0 deletions(-) Approvals: Merlijn van Deen: Looks good to me, approved jenkins-bot: Verified diff --git a/scripts/freebasemappingupload.py b/scripts/freebasemappingupload.py new file mode 100644 index 0000000..0bd2bba --- /dev/null +++ b/scripts/freebasemappingupload.py @@ -0,0 +1,103 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" +Script to upload the mappings of Freebase to Wikidata +Can be easily adapted to upload other String identifiers as well + +This bot needs the dump from +https://developers.google.com/freebase/data#freebase-wikidata-mappings + +The script takes a single parameter: + +-filename: the filename to read the freebase-wikidata mappings from; + default: fb2w.nt.gz +""" + +# +# (C) Denny Vrandecic, 2013 +# (C) Pywikibot team, 2013 +# Distributed under the terms of the MIT license. + +import gzip +import os +import sys + +import pywikibot + + +class FreebaseMapperRobot: + + def __init__(self, filename): + self.repo = pywikibot.Site('wikidata', 'wikidata').data_repository() + self.filename = filename + if not os.path.exists(self.filename): + pywikibot.output('Cannot find %s. Try providing the absolute path.' % self.filename) + sys.exit(1) + + def run(self): + # Set up some items we will use a lot. + self.claim = pywikibot.Claim(self.repo, 'P646') # freebase mapping + # And sources! + self.statedin = pywikibot.Claim(self.repo, 'P248') # stated in + freebasedumpitem = pywikibot.ItemPage(self.repo, 'Q15241312') # Freebase data dump + self.statedin.setTarget(freebasedumpitem) + self.dateofpub = pywikibot.Claim(self.repo, 'P577') # date of publication + oct28 = pywikibot.WbTime(year=2013, month=10, day=28, precision='day') + self.dateofpub.setTarget(oct28) + + for line in gzip.open(self.filename): + self.processLine(line.strip()) + + def processLine(self, line): + if not line or line.startswith('#'): + return + mid, sameas, qid, dot = line.split() + if sameas != '<http://www.w3.org/2002/07/owl#sameAs>': + return + if dot != '.': + return + if not mid.startswith('<http://rdf.freebase.com/ns/m'): + return + mid = '/m/' + mid[30:-1] + if not qid.startswith('<http://www.wikidata.org/entity/Q'): + return + qid = 'Q' + qid[33:-1] + data = pywikibot.ItemPage(self.repo, qid) + data.get() + if not data.labels: + label = '' + elif 'en' in data.labels: + label = data.labels['en'] + else: + # Just pick up the first label + label = data.labels.values()[0] + pywikibot.output('Parsed: %s <--> %s' % (qid, mid)) + pywikibot.output('%s is %s' % (data.getID(), label)) + if data.claims and 'P646' in data.claims: + # We assume that there is only one claim. + # If there are multiple ones, our logs might be wrong + # but the constraint value reports will catch them + if mid != data.claims['P646'][0].getTarget(): + pywikibot.output('Mismatch: expected %s, has %s instead' + % (mid, data.claims['P646'][0].getTarget())) + else: + pywikibot.output('Already has mid set, is consistent.') + else: + # No claim set, lets add it. + pywikibot.output('Going to add a new claim.') + self.claim.setTarget(mid) + data.addClaim(self.claim) + self.claim.addSources([self.statedin, self.dateofpub]) + pywikibot.output('Claim added!') + + +def main(): + filename = 'fb2w.nt.gz' # Default filename + for arg in pywikibot.handleArgs(): + if arg.startswith('-filename'): + filename = arg[11:] + bot = FreebaseMapperRobot(filename) + bot.run() + +if __name__ == '__main__': + main() -- To view, visit https://gerrit.wikimedia.org/r/99180 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I9d1f459d58da0eecd1631e8b863129d9d1aa3a85 Gerrit-PatchSet: 6 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Denny Vrandecic <vrandecic(a)gmail.com> Gerrit-Reviewer: Denny Vrandecic <vrandecic(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Legoktm <legoktm.wikipedia(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: jenkins-bot

1 0

[Gerrit] casechecker.py: ported to core - change (pywikibot/core)
by jenkins-bot (Code Review) 23 Dec '13

23 Dec '13

jenkins-bot has submitted this change and it was merged. Change subject: casechecker.py: ported to core ...................................................................... casechecker.py: ported to core Ported casechecker.py from pywikibot compat to pywikibot core Removed pywikibot.config.verbose_output, updated i18n scripts directory Change-Id: Iccb0720e09e2fa484800a779804fc49fff2ba3ec --- A scripts/casechecker.py 1 file changed, 829 insertions(+), 0 deletions(-) Approvals: Merlijn van Deen: Looks good to me, approved jenkins-bot: Verified diff --git a/scripts/casechecker.py b/scripts/casechecker.py new file mode 100644 index 0000000..98532bd --- /dev/null +++ b/scripts/casechecker.py @@ -0,0 +1,829 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +""" Script to enumerate all pages on the wiki and find all titles +with mixed latin and cyrilic alphabets. +""" +# +# (C) Pywikibot team, 2006-2013 +# +# Distributed under the terms of the MIT license. +# +__version__ = '$Id$' + +import os +import sys +import re +import codecs +import pywikibot +from pywikibot import i18n +from pywikibot.data import api + + +# +# Permutations code was taken from +# http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/190465 +# +def xuniqueCombinations(items, n): + if n == 0: + yield [] + else: + for i in xrange(len(items)): + for cc in xuniqueCombinations(items[i + 1:], n - 1): + yield [items[i]] + cc +# End of permutation code +# + +# +# Windows Concole colors +# This code makes this script Windows ONLY!!! +# Feel free to adapt it to another platform +# +# Adapted from http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/496901 +# +STD_OUTPUT_HANDLE = -11 + +FOREGROUND_BLUE = 0x01 # text color contains blue. +FOREGROUND_GREEN = 0x02 # text color contains green. +FOREGROUND_RED = 0x04 # text color contains red. +FOREGROUND_INTENSITY = 0x08 # text color is intensified. +BACKGROUND_BLUE = 0x10 # background color contains blue. +BACKGROUND_GREEN = 0x20 # background color contains green. +BACKGROUND_RED = 0x40 # background color contains red. +BACKGROUND_INTENSITY = 0x80 # background color is intensified. + +FOREGROUND_WHITE = FOREGROUND_BLUE | FOREGROUND_GREEN | FOREGROUND_RED + +try: + import ctypes + std_out_handle = ctypes.windll.kernel32.GetStdHandle(STD_OUTPUT_HANDLE) +except: + std_out_handle = None + + +def SetColor(color): + if std_out_handle: + try: + return ctypes.windll.kernel32.SetConsoleTextAttribute( + std_out_handle, color) + except: + pass + + if color == FOREGROUND_BLUE: + print '(b:', + if color == FOREGROUND_GREEN: + print '(g:', + if color == FOREGROUND_RED: + print '(r:', + +# end of console code + + +class CaseChecker(object): + # These words are always in one language, even though they could be typed + # in both + alwaysInLocal = [u'СССР', u'Как', u'как'] + alwaysInLatin = [u'II', u'III'] + + localUpperLtr = u'ЁІЇЎАБВГДЕЖЗИЙКЛМНОПРСТУФХЦЧШЩЪЫЬЭЮЯҐ' + localLowerLtr = u'ёіїўабвгдежзийклмнопрстуфхцчшщъыьэюяґ' + localLtr = localUpperLtr + localLowerLtr + + localSuspects = u'АВЕКМНОРСТХІЁЇаеорсухіёї' + latinSuspects = u'ABEKMHOPCTXIËÏaeopcyxiëï' + + # possibly try to fix one character mistypes in an alternative keyboard + # layout + localKeyboard = u'йцукенгшщзфывапролдячсмить' + latinKeyboard = u'qwertyuiopasdfghjklzxcvbnm' + + romanNumChars = u'IVXLMC' + # all letters that may be used as suffixes after roman numbers: "Iый" + romannumSuffixes = localLowerLtr + romanNumSfxPtrn = re.compile( + u'^[' + romanNumChars + ']+[' + localLowerLtr + ']+$') + + whitelists = { + 'ru': u'ВП:КЛ/Проверенные', + } + + latLtr = u'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ' + + lclClrFnt = u'' + latClrFnt = u'' + suffixClr = u'' + + wordBreaker = re.compile(u'[ _\-/\|#[\]():]') + stripChars = u' \t,' + + titles = True + links = False + aplimit = None + apfrom = u'' + title = None + replace = False + stopAfter = -1 + wikilog = None + wikilogfile = 'wikilog.txt' + failedTitles = 'failedTitles.txt' + nosuggestions = 'nosuggestions.txt' + doFailed = False + titleList = None + autonomous = False + namespaces = [] + filterredir = 'nonredirects' + + def __init__(self): + + for arg in pywikibot.handleArgs(): + if arg.startswith('-from'): + if arg.startswith('-from:'): + self.apfrom = arg[6:] + else: + self.apfrom = pywikibot.input(u'Which page to start from: ') + elif arg.startswith('-reqsize:'): + self.aplimit = int(arg[9:]) + elif arg == '-links': + self.links = True + elif arg == '-linksonly': + self.links = True + self.titles = False + elif arg == '-replace': + self.replace = True + elif arg == '-redir': + self.filterredir = 'all' + elif arg == '-redironly': + self.filterredir = 'redirects' + elif arg.startswith('-limit:'): + self.stopAfter = int(arg[7:]) + elif arg == '-autonomous' or arg == '-a': + self.autonomous = True + elif arg.startswith('-ns:'): + self.namespaces.append(int(arg[4:])) + elif arg.startswith('-wikilog:'): + self.wikilogfile = arg[9:] + elif arg.startswith('-failedlog:'): + self.failedTitles = arg[11:] + elif arg == '-failed': + self.doFailed = True + else: + pywikibot.output(u'Unknown argument %s.' % arg) + pywikibot.showHelp() + sys.exit() + + if self.namespaces == [] and not self.doFailed: + if self.apfrom == u'': + # 0 should be after templates ns + self.namespaces = [14, 10, 12, 0] + else: + self.namespaces = [0] + + if self.aplimit is None: + self.aplimit = 200 if self.links else 'max' + + if not self.doFailed: + self.queryParams = {'action': 'query', + 'generator': 'allpages', + 'gaplimit': self.aplimit, + 'gapfilterredir': self.filterredir} + else: + self.queryParams = {'action': 'query'} + if self.apfrom != u'': + pywikibot.output(u'Argument "-from" is ignored with "-failed"') + + propParam = 'info' + if self.links: + propParam += '|links|categories' + self.queryParams['pllimit'] = 'max' + self.queryParams['cllimit'] = 'max' + + self.queryParams['prop'] = propParam + + self.site = pywikibot.getSite() + + if len(self.localSuspects) != len(self.latinSuspects): + raise ValueError(u'Suspects must be the same size') + if len(self.localKeyboard) != len(self.latinKeyboard): + raise ValueError(u'Keyboard info must be the same size') + + if not os.path.isabs(self.wikilogfile): + self.wikilogfile = pywikibot.config.datafilepath(self.wikilogfile) + self.wikilog = self.OpenLogFile(self.wikilogfile) + + if not os.path.isabs(self.failedTitles): + self.failedTitles = pywikibot.config.datafilepath(self.failedTitles) + + if self.doFailed: + with codecs.open(self.failedTitles, 'r', 'utf-8') as f: + self.titleList = [self.Page(t) for t in f] + self.failedTitles += '.failed' + + self.lclToLatDict = dict([(ord(self.localSuspects[i]), + self.latinSuspects[i]) + for i in xrange(len(self.localSuspects))]) + self.latToLclDict = dict([(ord(self.latinSuspects[i]), + self.localSuspects[i]) + for i in xrange(len(self.localSuspects))]) + + if self.localKeyboard is not None: + self.lclToLatKeybDict = dict( + [(ord(self.localKeyboard[i]), + self.latinKeyboard[i]) + for i in xrange(len(self.localKeyboard))]) + self.latToLclKeybDict = dict( + [(ord(self.latinKeyboard[i]), + self.localKeyboard[i]) + for i in xrange(len(self.localKeyboard))]) + else: + self.lclToLatKeybDict = {} + self.latToLclKeybDict = {} + + badPtrnStr = u'([%s][%s]|[%s][%s])' \ + % (self.latLtr, self.localLtr, self.localLtr, self.latLtr) + self.badWordPtrn = re.compile(u'[%s%s]*%s[%s%s]*' + % (self.latLtr, self.localLtr, + badPtrnStr, self.latLtr, + self.localLtr)) + + # Get whitelist + self.knownWords = set() + self.seenUnresolvedLinks = set() + + # TODO: handle "continue" + if self.site.lang in self.whitelists: + wlpage = self.whitelists[self.site.lang] + pywikibot.output(u'Loading whitelist from %s' % wlpage) + wlparams = { + 'action': 'query', + 'prop': 'links', + 'titles': wlpage, + 'redirects': '', + 'indexpageids': '', + 'pllimit': 'max', + } + + req = api.Request(**wlparams) + data = req.submit() + if len(data['query']['pageids']) == 1: + pageid = data['query']['pageids'][0] + links = data['query']['pages'][pageid]['links'] + + allWords = [nn for n in links + for nn in self.FindBadWords(n['title'])] + + self.knownWords = set(allWords) +## kw = set() +## for w in allWords: +## if len(self.ProcessTitle(w)[1]) > 0: +## kw.add(w) +## self.knownWords = kw + + else: + raise ValueError(u'The number of pageids is not 1') + + pywikibot.output(u'Loaded whitelist with %i items' + % len(self.knownWords)) + if len(self.knownWords) > 0: + pywikibot.log(u'Whitelist: %s' + % u', '.join([self.MakeLink(i, False) + for i in self.knownWords])) + else: + pywikibot.output(u'Whitelist is not known for language %s' + % self.site.lang) + + def RunQuery(self, params): + while True: + # Get data + req = api.Request(**params) + data = req.submit() + + # Process received data + yield data + + # Clear any continuations first + if 'clcontinue' in params: + del params['clcontinue'] + if 'plcontinue' in params: + del params['plcontinue'] + + if 'query-continue' not in data: + if 'gapcontinue' in params: + del params['gapcontinue'] + break + + qc = data['query-continue'] + # First continue properties only, once done, continue with allpages + if 'categories' in qc or 'links' in qc: + if 'categories' in qc: + params.update(qc['categories']) + if 'links' in qc: + params.update(qc['links']) + elif 'allpages' in qc: + params.update(qc['allpages']) + else: + raise ValueError(u'Unexpected query-continue values: %s' % qc) + continue + + def Run(self): + try: + self.lastLetter = '' + + if not self.doFailed: + for namespace in self.namespaces: + self.currentTitle = None + self.queryParams['gapnamespace'] = namespace + self.queryParams['gapfrom'] = self.apfrom + for data in self.RunQuery(self.queryParams): + self.ProcessDataBlock(data) + else: + self.currentTitle = None + batchSize = 10 + for batchStart in xrange(0, len(self.titleList), batchSize): + self.queryParams['titles'] = self.titleList[ + batchStart:batchStart + batchSize] + for data in self.RunQuery(self.queryParams): + self.ProcessDataBlock(data) + print "*" * 29, "Done" + except: + pywikibot.output(u'Exception at Title = %s, Next = %s' + % (self.currentTitle, self.apfrom)) + try: + import traceback + pywikibot.output(traceback.format_exc()) + except: + pywikibot.output(u'Unable to print exception info') + raise + + def ProcessDataBlock(self, data): + if 'query' not in data or 'pages' not in data['query']: + return + + firstItem = True + for pageID, page in data['query']['pages'].iteritems(): + printed = False + title = page['title'] + self.currentTitle = title + if 'missing' in page: + continue + if firstItem: + if self.lastLetter != title[0]: + pywikibot.ui.output('Processing %s\n' % title) + self.lastLetter = title[0] + firstItem = False + if self.titles: + err = self.ProcessTitle(title) + if err: + changed = False + if self.replace: + if len(err[1]) == 1: + newTitle = err[1][0] +## choice = pywikibot.inputChoice(u'Move %s to %s?' +## % (title, newTitle), +## ['Yes', 'No'], +## ['y', 'n']) + editSummary = i18n.twtranslate(self.site, + "casechecker-rename") + dst = self.Page(newTitle) + + if 'redirect' in page: + src = self.Page(title) + redir = src.getRedirectTarget() + redirTitle = redir.title(asLink=True, + textlink=True) + + if not dst.exists(): + src.move(newTitle, editSummary, + movesubpages=True) + changed = True + + replErrors = False + for p in src.getReferences( + follow_redirects=False): + if p.namespace() == 2: + continue + oldText = p.get(get_redirect=True) + newText = self.ReplaceLink(oldText, title, + newTitle) + if not self.PutNewPage( + p, newText, [ + self.MakeMoveSummary(title, + newTitle)]): + replErrors = True + if not replErrors: + editSummary = i18n.twtranslate( + self.site, "casechecker-delete-summary") + newText = i18n.twtranslate( + self.site, + "casechecker-delete-reason", redirTitle, + fallback=False) + if newText: + src.put(newText, editSummary, + minorEdit=False) + changed = True + + elif not dst.exists(): + src = self.Page(title) + if page['ns'] == 14: + import category + dst = self.Page(newTitle) + bot = category.CategoryMoveRobot( + src.title(withNamespace=False), + dst.title(withNamespace=False), + self.autonomous, + editSummary + u' ' + + self.MakeMoveSummary(title, newTitle), + True) + bot.run() + else: + src.move(newTitle, editSummary, + movesubpages=True) + changed = True + + if not changed: + if len(err[1]) > 0: + self.AppendLineToLog(self.failedTitles, title) + else: + self.AddNoSuggestionTitle(title) + + self.WikiLog(u"* " + err[0]) + printed = True + + if self.links: + allLinks = None + if 'links' in page: + allLinks = page['links'] + if 'categories' in page: + if allLinks: + allLinks = allLinks + page['categories'] + else: + allLinks = page['categories'] + + if allLinks: + pageObj = None + pageTxt = None + msg = [] + foundSuggestions = False + + for l in allLinks: + ltxt = l['title'] + err = self.ProcessTitle(ltxt) + if err: + if len(err[1]) > 0: + foundSuggestions = True + elif self.AddNoSuggestionTitle(ltxt): + continue + + newTitle = None + if self.replace: + newTitle = self.PickTarget(title, ltxt, err[1]) + if newTitle: + if pageObj is None: + pageObj = self.Page(title) + pageTxt = pageObj.get() + + msg.append(self.MakeMoveSummary(ltxt, + newTitle)) + + pageTxt = self.ReplaceLink(pageTxt, ltxt, + newTitle) + if not newTitle: + if not printed: + self.WikiLog(u"* %s: link to %s" + % (self.MakeLink(title, False), + err[0])) + printed = True + else: + self.WikiLog(u"** link to %s" % err[0]) + if pageObj is not None: + if self.PutNewPage(pageObj, pageTxt, msg): + # done, no need to log anything + foundSuggestions = False + + if foundSuggestions: + self.AppendLineToLog(self.failedTitles, title) + if self.stopAfter > 0: + self.stopAfter -= 1 + if self.stopAfter == 0: + raise ValueError(u'Stopping because we are done') + + def WikiLog(self, text): + pywikibot.output(text) + self.wikilog.write(text + u'\n') + self.wikilog.flush() + + def FindBadWords(self, title): + for m in self.badWordPtrn.finditer(title): + yield title[m.span()[0]:m.span()[1]] + + def ProcessTitle(self, title): + badWords = list(self.FindBadWords(title)) + if len(badWords) > 0: + # Allow known words, allow any roman numerals with local suffixes + badWords = set([i for i in badWords + if i not in self.knownWords and + self.romanNumSfxPtrn.match(i) is not None]) + + if len(badWords) == 0 or self.Page(title).isImage(): + return + count = 0 + ambigBadWords = set() + ambigBadWordsCount = 0 + mapLcl = {} + mapLat = {} + + for badWord in badWords: + # See if it would make sense to treat the whole word as either + # cyrilic or latin + mightBeLat = mightBeLcl = True + for l in badWord: + if l in self.localLtr: + if mightBeLat and l not in self.localSuspects: + mightBeLat = False + else: + if mightBeLcl and l not in self.latinSuspects: + mightBeLcl = False + if l not in self.latLtr: + raise ValueError(u'Assert failed') + + # Some words are well known and frequently mixed-typed + if mightBeLcl and mightBeLat: + if badWord in self.alwaysInLocal: + mightBeLat = False + elif badWord in self.alwaysInLatin: + mightBeLoc = False + + if mightBeLcl: + mapLcl[badWord] = badWord.translate(self.latToLclDict) + if mightBeLat: + mapLat[badWord] = badWord.translate(self.lclToLatDict) + if mightBeLcl and mightBeLat: + ambigBadWords.add(badWord) + # Cannot do len(ambigBadWords) because they might be duplicates + ambigBadWordsCount += 1 + if not mightBeLcl and not mightBeLat: + # try to match one of the knownWords + bwLen = len(badWord) + kw = [w for w in self.knownWords if len(w) == bwLen] + for p in xrange(bwLen): + if len(kw) == 0: + break + c = badWord[p] + co = ord(c) + if co in self.latToLclDict: + c2 = self.latToLclDict[co] + elif co in self.lclToLatDict: + c2 = self.lclToLatDict[co] + else: + c2 = None + kw = [w for w in kw if p < len(w) and + (w[p] == c or (c2 is not None and w[p] == c2))] + if len(kw) > 1: + pywikibot.output(u"Word '%s' could be treated as more than " + u"one known words" % badWord) + elif len(kw) == 1: + mapLcl[badWord] = kw[0] + count += 1 + + infoText = self.MakeLink(title) + possibleAlternatives = [] + + if len(mapLcl) + len(mapLat) - ambigBadWordsCount < count: + # We cannot auto-translate - offer a list of suggested words + suggestions = mapLcl.values() + mapLat.values() + if len(suggestions) > 0: + infoText += u", word suggestions: " + u', '.join( + [self.ColorCodeWord(t) for t in suggestions]) + else: + infoText += u", no suggestions" + else: + + # Replace all unambiguous bad words + for k, v in mapLat.items() + mapLcl.items(): + if k not in ambigBadWords: + title = title.replace(k, v) + if len(ambigBadWords) == 0: + # There are no ambiguity, we can safelly convert + possibleAlternatives.append(title) + infoText += u", convert to " + self.MakeLink(title) + else: + # Try to pick 0, 1, 2, ..., len(ambiguous words) unique + # combinations from the bad words list, and convert just the + # picked words to cyrilic, whereas making all other words as + # latin character. + for itemCntToPick in xrange(0, len(ambigBadWords) + 1): + title2 = title + for uc in xuniqueCombinations(list(ambigBadWords), + itemCntToPick): + wordsToLat = ambigBadWords.copy() + for bw in uc: + title2 = title2.replace(bw, mapLcl[bw]) + wordsToLat.remove(bw) + for bw in wordsToLat: + title2 = title2.replace(bw, mapLat[bw]) + possibleAlternatives.append(title2) + + if len(possibleAlternatives) > 0: + infoText += u", can be converted to " + u', '.join( + [self.MakeLink(t) for t in possibleAlternatives]) + else: + infoText += u", no suggestions" + return (infoText, possibleAlternatives) + + def PickTarget(self, title, original, candidates): + if len(candidates) == 0: + return + if len(candidates) == 1: + return candidates[0] + + pagesDontExist = [] + pagesRedir = {} + pagesExist = [] + + for newTitle in candidates: + dst = self.Page(newTitle) + if not dst.exists(): + pagesDontExist.append(newTitle) + elif dst.isRedirectPage(): + pagesRedir[newTitle] = dst.getRedirectTarget().title() + else: + pagesExist.append(newTitle) + if len(pagesExist) == 1: + return pagesExist[0] + elif len(pagesExist) == 0 and len(pagesRedir) > 0: + if len(pagesRedir) == 1: + return pagesRedir.keys()[0] + t = None + for k, v in pagesRedir.iteritems(): + if not t: + t = v # first item + elif t != v: + break + else: + # all redirects point to the same target + # pick the first one, doesn't matter what it is + return pagesRedir.keys()[0] + + if not self.autonomous: + pywikibot.output(u'Could not auto-decide for page %s. Which link ' + u'should be chosen?' % self.MakeLink(title, False)) + pywikibot.output(u'Original title: ', newline=False) + self.ColorCodeWord(original + "\n", True) + count = 1 + for t in candidates: + if t in pagesDontExist: + msg = u'missing' + elif t in pagesRedir: + msg = u'Redirect to ' + pagesRedir[t] + else: + msg = u'page exists' + self.ColorCodeWord(u' %d: %s (%s)\n' % (count, t, msg), True) + count += 1 + answers = [str(i) for i in xrange(0, count)] + choice = int(pywikibot.inputChoice( + u'Which link to choose? (0 to skip)', + answers, [a[0] for a in answers])) + if choice > 0: + return candidates[choice - 1] + + def ColorCodeWord(self, word, toScreen=False): + if not toScreen: + res = u"" + lastIsCyr = word[0] in self.localLtr + if lastIsCyr: + if toScreen: + SetColor(FOREGROUND_GREEN) + else: + res += self.lclClrFnt + else: + if toScreen: + SetColor(FOREGROUND_RED) + else: + res += self.latClrFnt + + for l in word: + if l in self.localLtr: + if not lastIsCyr: + if toScreen: + SetColor(FOREGROUND_GREEN) + else: + res += self.suffixClr + self.lclClrFnt + lastIsCyr = True + elif l in self.latLtr: + if lastIsCyr: + if toScreen: + SetColor(FOREGROUND_RED) + else: + res += self.suffixClr + self.latClrFnt + lastIsCyr = False + if toScreen: + pywikibot.output(l, newline=False) + else: + res += l + + if toScreen: + SetColor(FOREGROUND_WHITE) + else: + return res + self.suffixClr + u"" + + def AddNoSuggestionTitle(self, title): + if title in self.seenUnresolvedLinks: + return True + self.seenUnresolvedLinks.add(title) + + params = { + 'action': 'query', + 'list': 'backlinks', + 'bltitle': title, + 'bllimit': '50', + } + + req = api.Request(**params) + data = req.submit() + cl = 0 + redirs = 0 + if 'backlinks' in data['query']: + bl = data['query']['backlinks'] + cl = len(bl) + redirs = len([i for i in bl if 'redirect' in i]) + + if cl > 0 and 'query-continue' in data: + count = '50+' + else: + count = str(cl if cl > 0 else 'no backlinks') + + self.AppendLineToLog(self.nosuggestions, u'* %s (%s%s)' + % (self.MakeLink(title), count, u', %d redirects' + % redirs if redirs > 0 else u'')) + return False + + def PutNewPage(self, pageObj, pageTxt, msg): + title = pageObj.title(asLink=True, textlink=True) + coloredMsg = u', '.join([self.ColorCodeWord(m) for m in msg]) + if pageObj.get(get_redirect=True) == pageTxt: + self.WikiLog(u"* Error: Text replacement failed in %s (%s)" + % (self.MakeLink(title, False), coloredMsg)) + else: + pywikibot.output(u'Case Replacements: %s' % u', '.join(msg)) + try: + pageObj.put( + pageTxt, + u'%s: %s' + % (i18n.twtranslate( + self.site, + "casechecker-replacement-summary"), + self.site.mediawiki_message(u"Comma-separator").join(msg))) + return True + except KeyboardInterrupt: + raise + except: + self.WikiLog(u"* Error: Could not save updated page %s (%s)" + % (self.MakeLink(title, False), coloredMsg)) + return False + + def MakeMoveSummary(self, fromTitle, toTitle): + return i18n.twtranslate(self.site, "casechecker-replacement-linklist") % {'source': fromTitle, 'target': toTitle} + + def MakeLink(self, title, colorcode=True): + prf = u'' if self.Page(title).namespace() == 0 else u':' + cc = u'|««« %s »»»' % self.ColorCodeWord(title) if colorcode else u'' + return u"[[%s%s%s]]" % (prf, title, cc) + + def OpenLogFile(self, filename): + try: + return codecs.open(filename, 'a', 'utf-8') + except IOError: + return codecs.open(filename, 'w', 'utf-8') + + def AppendLineToLog(self, filename, text): + with self.OpenLogFile(filename) as f: + f.write(text + u'\n') + + def Page(self, title): + return pywikibot.Page(self.site, title) + + def ReplaceLink(self, text, oldtxt, newtxt): + + frmParts = [s.strip(self.stripChars) + for d in self.wordBreaker.split(oldtxt)] + toParts = [s.strip(self.stripChars) + for s in self.wordBreaker.split(newtxt)] + + if len(frmParts) != len(toParts): + raise ValueError(u'Splitting parts do not match counts') + for i in xrange(0, len(frmParts)): + if len(frmParts[i]) != len(toParts[i]): + raise ValueError(u'Splitting parts do not match word length') + if len(frmParts[i]) > 0: + text = text.replace(frmParts[i][0].lower() + frmParts[i][1:], + toParts[i][0].lower() + toParts[i][1:]) + text = text.replace(frmParts[i][0].upper() + frmParts[i][1:], + toParts[i][0].upper() + toParts[i][1:]) + + return text + + +if __name__ == "__main__": + try: + bot = CaseChecker() + bot.Run() + finally: + pywikibot.stopme() -- To view, visit https://gerrit.wikimedia.org/r/101609 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Iccb0720e09e2fa484800a779804fc49fff2ba3ec Gerrit-PatchSet: 9 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Jcf2000 <jaredflores2000(a)gmail.com> Gerrit-Reviewer: Jcf2000 <jaredflores2000(a)gmail.com> Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com> Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl> Gerrit-Reviewer: Strainu <wiki(a)strainu.ro> Gerrit-Reviewer: Xqt <info(a)gno.de> Gerrit-Reviewer: Yurik <yurik(a)wikimedia.org> Gerrit-Reviewer: jenkins-bot

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

Pywikibot-commits