Revision: 5844
Author: cosoleto
Date: 2008-08-24 17:48:31 +0000 (Sun, 24 Aug 2008)
Log Message:
-----------
Applied patch #2070928, ja translations for category_redirect.py by Nightshadow28
Modified Paths:
--------------
trunk/pywikipedia/category_redirect.py
Modified: trunk/pywikipedia/category_redirect.py
===================================================================
--- trunk/pywikipedia/category_redirect.py 2008-08-24 11:59:21 UTC (rev 5843)
+++ trunk/pywikipedia/category_redirect.py 2008-08-24 17:48:31 UTC (rev 5844)
@@ -1,654 +1,662 @@
-# -*- coding: utf-8 -*-
-"""This bot will move pages out of redirected categories
-
-Usage: category-redirect.py [options]
-
-The following command-line options can be used with this bot:
-
--often Only scan those redirected categories that have been
- identified as often-populated (only useful if the site
- has such a category defined)
-
-"""
-
-import wikipedia, catlib
-import pagegenerators
-import simplejson
-import cPickle
-import math
-import re
-import sys, traceback
-import threading, Queue
-import time
-from datetime import datetime, timedelta
-
-class APIError(Exception):
- """The wiki API returned an error message."""
- def __init__(self, errordict):
- """Save error dict returned by MW API."""
- self.errors = errordict
-
- def __str__(self):
- return "%(code)s: %(info)s" % self.errors
-
-
-class ThreadList(list):
- """A simple threadpool class to limit the number of simultaneous threads.
-
- Any threading.Thread object can be added to the pool using the append()
- method. If the maximum number of simultaneous threads has not been
- reached, the Thread object will be started immediately; if not, the
- append() call will block until the thread is able to start.
-
- >>> pool = ThreadList(limit=10)
- >>> def work():
- ... time.sleep(1)
- ...
- >>> for x in xrange(20):
- ... pool.append(threading.Thread(target=work))
- ...
-
- """
- def __init__(self, limit=sys.maxint, *args):
- self.limit = limit
- list.__init__(self, *args)
- for item in list(self):
- if not isinstance(threading.Thread, item):
- raise TypeError("Cannot add '%s' to ThreadList" % type(item))
-
- def active_count(self):
- """Return the number of alive threads, and delete all non-alive ones."""
- count = 0
- for item in list(self):
- if item.isAlive():
- count += 1
- else:
- self.remove(item)
- return count
-
- def append(self, thd):
- if not isinstance(thd, threading.Thread):
- raise TypeError("Cannot append '%s' to ThreadList" % type(thd))
- while self.active_count() >= self.limit:
- time.sleep(2)
- list.append(self, thd)
- thd.start()
-
-
-class CategoryRedirectBot(object):
- def __init__(self, often):
- self.cooldown = 6 # days
- self.often = often
-
- # Localization:
-
- # Category that contains all redirected category pages
- self.cat_redirect_cat = {
- 'wikipedia': {
- 'en': "Category:Wikipedia category redirects",
- 'ar': "تصنيف:تحويلات تصنيفات ويكيبيديا",
- 'no': "Kategori:Wikipedia omdirigertekategorier",
- 'simple': "Category:Category redirects",
- },
- 'commons': {
- 'commons': "Category:Non-empty category redirects"
- }
- }
-
- # Category that contains frequently-used redirected category pages
- self.often_redirect_cat = {
- 'wikipedia': {
- 'en': "Category:Often-populated Wikipedia category redirects",
- },
- }
-
- # List of all templates that are used to mark category redirects
- # (put the most preferred form first)
- self.redir_templates = {
- 'wikipedia': {
- 'en': ("Category redirect",
- "Category redirect3",
- "Categoryredirect",
- "Empty category",
- "CR",
- "Catredirect",
- "Emptycat",
- "Emptycategory",
- "Empty cat",
- "Seecat",),
- 'ar': ("تحويل تصنيف",
- "Category redirect",
- "تحويلة تصنيف",),
- 'no': ("Kategoriomdirigering",),
- 'simple': ("Category redirect",
- "Catredirect"),
- },
- 'commons': {
- 'commons': (u'Category redirect',
- u'Categoryredirect',
- u'See cat',
- u'Seecat',
- u'Catredirect',
- u'Cat redirect',
- u'CatRed',
- u'Catredir',),
- }
- }
-
- self.move_comment = {
- '_default':
-u"Robot: moving pages out of redirected category",
- 'ar':
-u"روبوت: نقل الصفحات من تصنيف محول",
- 'no':
-u"Robot: Flytter sider ut av omdirigeringskategori",
- 'commons':
-u'Robot: Changing category link (following [[Template:Category redirect|category redirect]])'
- }
-
- self.redir_comment = {
- '_default':
-u"Robot: adding category redirect template for maintenance",
- 'ar':
-u"روبوت: إضافة قالب تحويل تصنيف للصيانة",
- 'no':
-u"Robot: Legger til vedlikeholdsmal for kategoriomdirigering",
- }
-
- self.dbl_redir_comment = {
- '_default': u"Robot: fixing double-redirect",
- 'ar': u"روبوت: تصليح تحويلة مزدوجة",
- 'no': u"Robot: Ordner doble omdirigeringer",
- }
-
- self.maint_comment = {
- '_default': u"Category redirect maintenance bot",
- 'ar': u"بوت صيانة تحويل التصنيف",
- 'no': u"Bot for vedlikehold av kategoriomdirigeringer",
- }
-
- def change_category(self, article, oldCat, newCat, comment=None,
- sortKey=None):
- """Given an article in category oldCat, moves it to category newCat.
- Moves subcategories of oldCat as well. oldCat and newCat should be
- Category objects. If newCat is None, the category will be removed.
-
- This is a copy of portions of catlib.change_category() with the
- added capability to post a talk page message on pages that cannot be
- fixed due to protection.
-
- """
- cats = article.categories(get_redirect=True)
-
- oldtext = article.get(get_redirect=True)
- newtext = wikipedia.replaceCategoryInPlace(oldtext, oldCat, newCat)
- if newtext == oldtext:
- wikipedia.output(
- u'No changes in made in page %s.' % article.aslink())
- return False
- try:
- article.put(newtext, comment)
- return True
- except wikipedia.EditConflict:
- wikipedia.output(
- u'Skipping %s because of edit conflict' % article.aslink())
- except wikipedia.LockedPage:
- wikipedia.output(u'Skipping locked page %s' % article.aslink())
- if not article.isTalkPage and article.namespace != 2:
- # no messages on user pages or non-talk pages
- talkpage = article.toggleTalkPage()
- try:
- talktext = talk.get()
- except wikipedia.IsRedirectPage:
- return False
- except wikipedia.NoPage:
- talktext = u""
- if not talk.isTalkPage():
- return False
- talktext = talktext + u"""
-== Category link ==
-{{editprotected}}
-* This protected page has been detected in [[%s]], but that category has \
-been redirected to [[%s]]. Please update the category link. --~~~~
-""" % (oldCat.aslink(textlink=True), newCat.aslink(textlink=True))
- # NEEDS LOCALIZATION
- try:
- talkpage.put(talktext,
- u"Robot: Category-redirect notification on protected page",
- minorEdit=False) # NEEDS LOCALIZATION
- wikipedia.output(
- u"Left protected page notification on %s"
- % talkpage.aslink())
- except wikipedia.PageNotSaved:
- wikipedia.output(
- u"Protected page notification on %s failed"
- % talkpage.aslink())
-
- except wikipedia.SpamfilterError, error:
- wikipedia.output(
- u'Changing page %s blocked by spam filter (URL=%s)'
- % (article.aslink(), error.url))
- except wikipedia.NoUsername:
- wikipedia.output(
- u"Page %s not saved; sysop privileges required."
- % article.aslink())
- except wikipedia.PageNotSaved, error:
- wikipedia.output(u"Saving page %s failed: %s"
- % (article.aslink(), error.message))
- return False
-
- def move_contents(self, oldCatTitle, newCatTitle, editSummary):
- """The worker function that moves pages out of oldCat into newCat"""
- while True:
- try:
- oldCat = catlib.Category(self.site,
- self.catprefix + oldCatTitle)
- newCat = catlib.Category(self.site,
- self.catprefix + newCatTitle)
-
- # Move articles
- found, moved = 0, 0
- for result in self.query_results(list="categorymembers",
- cmtitle=oldCat.title(),
- cmprop="title|sortkey",
- cmlimit="max"):
- found += len(result['categorymembers'])
- for item in result['categorymembers']:
- article = wikipedia.Page(self.site, item['title'])
- changed = self.change_category(article, oldCat, newCat,
- comment=editSummary)
- if changed: moved += 1
-
- # pass 2: look for template doc pages
- for result in self.query_results(list="categorymembers",
- cmtitle=oldCat.title(),
- cmprop="title|sortkey",
- cmnamespace="10",
- cmlimit="max"):
- for item in result['categorymembers']:
- doc = wikipedia.Page(self.site, item['title']+"/doc")
- try:
- old_text = doc.get()
- except wikipedia.Error:
- continue
- changed = self.change_category(article, oldCat, newCat,
- comment=editSummary)
- if changed: moved += 1
-
- if found:
- wikipedia.output(u"%s: %s found, %s moved"
- % (oldCat.title(), found, moved))
- #Dummy edit to refresh the page, shouldn't show up in any logs.
- try:
- oldCat.put(oldCat.get())
- except:
- self.log_text.append(u'* Dummy edit at %s failed'
- % oldCat.aslink(textlink=True))
- self.result_queue.put((oldCatTitle, found, moved))
- return
- except wikipedia.ServerError:
- wikipedia.output(u"Server error: retrying in 5 seconds...")
- time.sleep(5)
- continue
- except:
- self.result_queue.put((oldCatTitle, None, None))
- raise
-
- def readyToEdit(self, cat):
- """Return True if cat not edited during cooldown period, else False."""
- dateformat ="%Y%m%d%H%M%S"
- today = datetime.now()
- deadline = today + timedelta(days=-self.cooldown)
- if cat.editTime() is None:
- raise RuntimeError
- return (deadline.strftime(dateformat) > cat.editTime())
-
- def query_results(self, **data):
- """Iterate results from API action=query, using data as parameters."""
- addr = self.site.apipath()
- querydata = {'action': 'query',
- 'format': 'json',
- 'maxlag': str(wikipedia.config.maxlag)}
- querydata.update(data)
- if not querydata.has_key("action")\
- or not querydata['action'] == 'query':
- raise ValueError(
- "query_results: 'action' set to value other than 'query'"
- )
- waited = 0
- while True:
- response, data = self.site.postForm(addr, querydata)
- if data.startswith(u"unknown_action"):
- e = {'code': data[:14], 'info': data[16:]}
- raise APIError(e)
- try:
- result = simplejson.loads(data)
- except ValueError:
- # if the result isn't valid JSON, there must be a server
- # problem. Wait a few seconds and try again
- # TODO: warn user; if the server is down, this could
- # cause an infinite loop
- wikipedia.output("Invalid API response received; retrying...")
- time.sleep(5)
- continue
- if type(result) is dict and result.has_key("error"):
- if result['error']['code'] == "maxlag":
- print "Pausing due to server lag.\r",
- time.sleep(5)
- waited += 5
- if waited % 30 == 0:
- wikipedia.output(
- u"(Waited %i seconds due to server lag.)"
- % waited)
- continue
- else:
- # raise error
- raise APIError(result['error'])
- waited = 0
- if type(result) is list:
- # query returned no results
- return
- assert type(result) is dict, \
- "Unexpected result of type '%s' received." % type(result)
- assert result.has_key("query"), \
- "No 'query' response found, result keys = %s" % result.keys()
- yield result['query']
- if result.has_key("query-continue"):
- assert len(result['query-continue'].keys()) == 1, \
- "More than one query-continue key returned: %s" \
- % result['query-continue'].keys()
- query_type = result['query-continue'].keys()[0]
- assert (query_type in querydata.keys()
- or query_type in querydata.values()), \
- "Site returned unknown query-continue type '%s'"\
- % query_type
- querydata.update(result['query-continue'][query_type])
- else:
- return
-
- def run(self):
- """Run the bot"""
- self.site = wikipedia.getSite()
- self.catprefix = self.site.namespace(14)+":"
- self.result_queue = Queue.Queue()
- self.log_text = []
-
- user = self.site.loggedInAs()
- redirect_magicwords = ["redirect"]
- other_words = self.site.redirect()
- if other_words:
- redirect_magicwords.extend(other_words)
-
- problems = []
-
- problem_page = wikipedia.Page(self.site,
- u"User:%(user)s/category redirect problems" % locals())
- l = time.localtime()
- today = "%04d-%02d-%02d" % l[:3]
- log_page = wikipedia.Page(self.site,
- u"User:%(user)s/category redirect logs/%(today)s"
- % locals())
-
- datafile = wikipedia.config.datafilepath(
- "%s-catmovebot-data" % self.site.dbName())
- try:
- inp = open(datafile, "rb")
- record = cPickle.load(inp)
- inp.close()
- except IOError:
- record = {}
- if record:
- cPickle.dump(record, open(datafile + ".bak", "wb"))
-
- # Set up regexes for later scanning
- template_list = self.redir_templates[self.site.family.name
- ][self.site.lang]
- # regex to match soft category redirects
- template_regexes = [
- re.compile(
-ur"{{\s*(?:%(prefix)s\s*:\s*)?%(template)s\s*\|(\s*%(catns)s\s*:\s*)?([^}]+)}}"
- % {'prefix': self.site.namespace(10).lower(),
- 'template': item.replace(" ", "[ _]+"),
- 'catns': self.site.namespace(14)},
- re.I)
- for item in template_list
- ]
- template_regex = re.compile(
-ur"{{\s*(?:%(prefix)s\s*:\s*)?(?:%(template)s)\s*\|(\s*%(catns)s\s*:\s*)?([^}]+)}}"
- % {'prefix': self.site.namespace(10).lower(),
- 'template': "|".join(item.replace(" ", "[ _]+")
- for item in template_list),
- 'catns': self.site.namespace(14)},
- re.I)
- # regex to match hard redirects to category pages
- catredir_regex = re.compile(
-ur'\s*#(?:%(redir)s)\s*:?\s*\[\[\s*:?%(catns)s\s*:(.*?)\]\]\s*'
- % {'redir': "|".join(redirect_magicwords),
- 'catns': self.site.namespace(14)},
- re.I)
- # regex to match all other hard redirects
- redir_regex = re.compile(ur"(?i)\s*#(?:%s)\s*:?\s*\[\[(.*?)\]\]"
- % "|".join(redirect_magicwords),
- re.I)
-
- # check for hard-redirected categories that are not already marked
- # with an appropriate template
- comment = wikipedia.translate(self.site.lang, self.redir_comment)
- for result in self.query_results(list='allpages',
- apnamespace='14', # Category:
- apfrom='!',
- apfilterredir='redirects',
- aplimit='max'):
- gen = (wikipedia.Page(self.site, page_item['title'])
- for page_item in result['allpages'])
- for page in pagegenerators.PreloadingGenerator(gen, 120):
- text = page.get(get_redirect=True)
- if re.search(template_regex, text):
- # this is already a soft-redirect, so skip it (for now)
- continue
- m = catredir_regex.match(text)
- if m:
- # this is a hard-redirect to a category page
- newtext = (u"{{%(template)s|%(cat)s}}"
- % {'cat': m.group(1),
- 'template': template_list[0]})
- try:
- page.put(newtext, comment, minorEdit=True)
- self.log_text.append(u"* Added {{tl|%s}} to %s"
- % (template_list[0],
- page.aslink(textlink=True)))
- except wikipedia.Error, e:
- self.log_text.append(
- u"* Failed to add {{tl|%s}} to %s (%s)"
- % (template_list[0],
- page.aslink(textlink=True),
- e))
- else:
- r = redir_regex.match(text)
- if r:
- problems.append(
- u"# %s is a hard redirect to [[:%s]]"
- % (page.aslink(textlink=True),
- r.group(1)))
- else:
- problems.append(
- u"# %s is a hard redirect; unable to extract target."
- % page.aslink(textlink=True))
-
- wikipedia.output("Done checking hard-redirect category pages.")
-
- comment = wikipedia.translate(self.site.lang, self.move_comment)
- scan_data = {
- u'action': 'query',
- u'list': 'embeddedin',
- u'einamespace': '14', # Category:
- u'eilimit': 'max',
- u'format': 'json'
- }
- counts = {}
- destmap = {}
- catmap = {}
- catlist = []
- catpages = []
- if self.often:
- target = self.often_redirect_cat[self.site.family.name
- ][self.site.lang]
- else:
- target = self.cat_redirect_cat[self.site.family.name
- ][self.site.lang]
-
- # get and preload all members of the category-redirect category
- for result in self.query_results(generator=u'categorymembers',
- gcmtitle=target,
- gcmnamespace=u'14', # CATEGORY
- gcmlimit=u'max',
- prop='info'):
- for catdata in result['pages'].values():
- catpages.append(wikipedia.Page(self.site, catdata['title']))
-
- wikipedia.output(u"")
- wikipedia.output(u"Preloading %s category redirect pages"
- % len(catpages))
- thread_limit = int(math.log(len(catpages), 2))
- for cat in pagegenerators.PreloadingGenerator(catpages, 120):
- cat_title = cat.titleWithoutNamespace()
- if "Wikipedia category redirect" in cat_title:
-# self.log_text.append("* Ignoring %s%s" % (self.catprefix, cat_title))
- continue
- try:
- text = cat.get(get_redirect=True)
- except wikipedia.Error:
- self.log_text.append(u"* Could not load %s%s; ignoring"
- % (self.catprefix, cat_title))
- continue
- match = template_regex.search(text)
- if match is None:
- self.log_text.append(u"* False positive: %s" % cat_title)
- continue
- catlist.append(cat)
- destination = match.group(2)
- target = catlib.Category(self.site, self.catprefix+destination)
- destmap.setdefault(target, []).append(cat)
- catmap[cat] = destination
- if match.group(1):
- # category redirect target starts with "Category:" - fix it
- text = text[ :match.start(1)] + text[match.end(1): ]
- cat.put(text,
- u"Robot: fixing category redirect parameter format")
- self.log_text.append(
- u"* Removed category prefix from parameter in %s"
- % cat.aslink(textlink=True))
-
- wikipedia.output(u"")
- wikipedia.output(u"Checking %s destination categories" % len(destmap))
- for dest in pagegenerators.PreloadingGenerator(destmap.keys(), 120):
- if not dest.exists():
- for d in destmap[dest]:
- problems.append("# %s redirects to %s"
- % (d.aslink(textlink=True),
- dest.aslink(textlink=True)))
- catlist.remove(d)
- # do a null edit on d to make it appear in the
- # "needs repair" category (if this wiki has one)
- try:
- d.put(d.get(get_redirect=True))
- except:
- pass
- if dest in catlist:
- for d in destmap[dest]:
- # is catmap[dest] also a redirect?
- newcat = catlib.Category(self.site,
- self.catprefix+catmap[dest])
- while newcat in catlist:
- if newcat == d or newcat == dest:
- self.log_text.append(u"* Redirect loop from %s"
- % newcat.aslink(textlink=True))
- break
- newcat = catlib.Category(self.site,
- self.catprefix+catmap[newcat])
- else:
- self.log_text.append(
- u"* Fixed double-redirect: %s -> %s -> %s"
- % (d.aslink(textlink=True),
- dest.aslink(textlink=True),
- newcat.aslink(textlink=True)))
- oldtext = d.get(get_redirect=True)
- # remove the old redirect from the old text,
- # leaving behind any non-redirect text
- oldtext = template_regex.sub("", oldtext)
- newtext = (u"{{category redirect|%(ncat)s}}"
- % {'ncat': newcat.titleWithoutNamespace()})
- newtext = newtext + oldtext.strip()
- try:
- d.put(newtext,
- wikipedia.translate(self.site.lang,
- self.dbl_redir_comment),
- minorEdit=True)
- except wikipedia.Error, e:
- self.log_text.append("** Failed: %s" % str(e))
-
- wikipedia.output(u"")
- wikipedia.output(u"Moving pages out of %s redirected categories."
- % len(catlist))
- threadpool = ThreadList(limit=thread_limit)
-
- for cat in catlist:
- cat_title = cat.titleWithoutNamespace()
- if not self.readyToEdit(cat):
- counts[cat_title] = None
- continue
- threadpool.append(threading.Thread(target=self.move_contents,
- args=(cat_title, catmap[cat]),
- kwargs=dict(editSummary=comment)))
- while len(counts) < len(catlist):
- title, found, moved = self.result_queue.get()
- if found is None:
- self.log_text.append(
- u"* [[:%s%s]]: error in move_contents thread"
- % (self.catprefix, title))
- else:
- if found:
- self.log_text.append(
- u"* [[:%s%s]]: %d found, %d moved"
- % (self.catprefix, title, found, moved))
- counts[title] = found
-
- if not self.often:
- for cat in record.keys():
- if cat not in counts.keys():
- del record[cat]
- for cat in counts.keys():
- if counts[cat] is not None:
- if counts[cat]:
- record.setdefault(cat, {})[today] = counts[cat]
- else:
- record.setdefault(cat, {})
- cPickle.dump(record, open(datafile, "wb"))
-
- wikipedia.setAction(wikipedia.translate(self.site.lang,
- self.maint_comment))
- log_page.put("\n".join(self.log_text))
- problem_page.put("\n".join(problems))
-
-
-def main(*args):
- try:
- a = wikipedia.handleArgs(*args)
- if "-often" in a:
- a.remove("-often")
- often = True
- else:
- often = False
- if len(a) == 1:
- raise RuntimeError('Unrecognized argument "%s"' % a[0])
- elif a:
- raise RuntimeError('Unrecognized arguments: ' +
- " ".join(('"%s"' % arg) for arg in a))
- bot = CategoryRedirectBot(often)
- bot.run()
- finally:
- wikipedia.stopme()
-
-
-if __name__ == "__main__":
- main()
+# -*- coding: utf-8 -*-
+"""This bot will move pages out of redirected categories
+
+Usage: category-redirect.py [options]
+
+The following command-line options can be used with this bot:
+
+-often Only scan those redirected categories that have been
+ identified as often-populated (only useful if the site
+ has such a category defined)
+
+"""
+
+import wikipedia, catlib
+import pagegenerators
+import simplejson
+import cPickle
+import math
+import re
+import sys, traceback
+import threading, Queue
+import time
+from datetime import datetime, timedelta
+
+class APIError(Exception):
+ """The wiki API returned an error message."""
+ def __init__(self, errordict):
+ """Save error dict returned by MW API."""
+ self.errors = errordict
+
+ def __str__(self):
+ return "%(code)s: %(info)s" % self.errors
+
+
+class ThreadList(list):
+ """A simple threadpool class to limit the number of simultaneous threads.
+
+ Any threading.Thread object can be added to the pool using the append()
+ method. If the maximum number of simultaneous threads has not been
+ reached, the Thread object will be started immediately; if not, the
+ append() call will block until the thread is able to start.
+
+ >>> pool = ThreadList(limit=10)
+ >>> def work():
+ ... time.sleep(1)
+ ...
+ >>> for x in xrange(20):
+ ... pool.append(threading.Thread(target=work))
+ ...
+
+ """
+ def __init__(self, limit=sys.maxint, *args):
+ self.limit = limit
+ list.__init__(self, *args)
+ for item in list(self):
+ if not isinstance(threading.Thread, item):
+ raise TypeError("Cannot add '%s' to ThreadList" % type(item))
+
+ def active_count(self):
+ """Return the number of alive threads, and delete all non-alive ones."""
+ count = 0
+ for item in list(self):
+ if item.isAlive():
+ count += 1
+ else:
+ self.remove(item)
+ return count
+
+ def append(self, thd):
+ if not isinstance(thd, threading.Thread):
+ raise TypeError("Cannot append '%s' to ThreadList" % type(thd))
+ while self.active_count() >= self.limit:
+ time.sleep(2)
+ list.append(self, thd)
+ thd.start()
+
+
+class CategoryRedirectBot(object):
+ def __init__(self, often):
+ self.cooldown = 6 # days
+ self.often = often
+
+ # Localization:
+
+ # Category that contains all redirected category pages
+ self.cat_redirect_cat = {
+ 'wikipedia': {
+ 'en': "Category:Wikipedia category redirects",
+ 'ar': "تصنيف:تحويلات تصنيفات ويكيبيديا",
+ 'no': "Kategori:Wikipedia omdirigertekategorier",
+ 'ja': "Category:移行中のカテゴリ",
+ 'simple': "Category:Category redirects",
+ },
+ 'commons': {
+ 'commons': "Category:Non-empty category redirects"
+ }
+ }
+
+ # Category that contains frequently-used redirected category pages
+ self.often_redirect_cat = {
+ 'wikipedia': {
+ 'en': "Category:Often-populated Wikipedia category redirects",
+ },
+ }
+
+ # List of all templates that are used to mark category redirects
+ # (put the most preferred form first)
+ self.redir_templates = {
+ 'wikipedia': {
+ 'en': ("Category redirect",
+ "Category redirect3",
+ "Categoryredirect",
+ "Empty category",
+ "CR",
+ "Catredirect",
+ "Emptycat",
+ "Emptycategory",
+ "Empty cat",
+ "Seecat",),
+ 'ar': ("تحويل تصنيف",
+ "Category redirect",
+ "تحويلة تصنيف",),
+ 'no': ("Kategoriomdirigering",),
+ 'ja': ("Category redirect",),
+ 'simple': ("Category redirect",
+ "Catredirect"),
+ },
+ 'commons': {
+ 'commons': (u'Category redirect',
+ u'Categoryredirect',
+ u'See cat',
+ u'Seecat',
+ u'Catredirect',
+ u'Cat redirect',
+ u'CatRed',
+ u'Catredir',),
+ }
+ }
+
+ self.move_comment = {
+ '_default':
+u"Robot: moving pages out of redirected category",
+ 'ar':
+u"روبوت: نقل الصفحات من تصنيف محول",
+ 'no':
+u"Robot: Flytter sider ut av omdirigeringskategori",
+ 'ja':
+u"ロボットによる: 移行中のカテゴリからのカテゴリ変更",
+ 'commons':
+u'Robot: Changing category link (following [[Template:Category redirect|category redirect]])'
+ }
+
+ self.redir_comment = {
+ '_default':
+u"Robot: adding category redirect template for maintenance",
+ 'ar':
+u"روبوت: إضافة قالب تحويل تصنيف للصيانة",
+ 'ja':
+u"ロボットによる: 移行中のカテゴリとしてタグ付け",
+ 'no':
+u"Robot: Legger til vedlikeholdsmal for kategoriomdirigering",
+ }
+
+ self.dbl_redir_comment = {
+ '_default': u"Robot: fixing double-redirect",
+ 'ar': u"روبوت: تصليح تحويلة مزدوجة",
+ 'ja': u"ロボットによる: 二重リダイレクト修正",
+ 'no': u"Robot: Ordner doble omdirigeringer",
+ }
+
+ self.maint_comment = {
+ '_default': u"Category redirect maintenance bot",
+ 'ar': u"بوت صيانة تحويل التصنيف",
+ 'ja': u"移行中のカテゴリのメンテナンス・ボット",
+ 'no': u"Bot for vedlikehold av kategoriomdirigeringer",
+ }
+
+ def change_category(self, article, oldCat, newCat, comment=None,
+ sortKey=None):
+ """Given an article in category oldCat, moves it to category newCat.
+ Moves subcategories of oldCat as well. oldCat and newCat should be
+ Category objects. If newCat is None, the category will be removed.
+
+ This is a copy of portions of catlib.change_category() with the
+ added capability to post a talk page message on pages that cannot be
+ fixed due to protection.
+
+ """
+ cats = article.categories(get_redirect=True)
+
+ oldtext = article.get(get_redirect=True)
+ newtext = wikipedia.replaceCategoryInPlace(oldtext, oldCat, newCat)
+ if newtext == oldtext:
+ wikipedia.output(
+ u'No changes in made in page %s.' % article.aslink())
+ return False
+ try:
+ article.put(newtext, comment)
+ return True
+ except wikipedia.EditConflict:
+ wikipedia.output(
+ u'Skipping %s because of edit conflict' % article.aslink())
+ except wikipedia.LockedPage:
+ wikipedia.output(u'Skipping locked page %s' % article.aslink())
+ if not article.isTalkPage and article.namespace != 2:
+ # no messages on user pages or non-talk pages
+ talkpage = article.toggleTalkPage()
+ try:
+ talktext = talk.get()
+ except wikipedia.IsRedirectPage:
+ return False
+ except wikipedia.NoPage:
+ talktext = u""
+ if not talk.isTalkPage():
+ return False
+ talktext = talktext + u"""
+== Category link ==
+{{editprotected}}
+* This protected page has been detected in [[%s]], but that category has \
+been redirected to [[%s]]. Please update the category link. --~~~~
+""" % (oldCat.aslink(textlink=True), newCat.aslink(textlink=True))
+ # NEEDS LOCALIZATION
+ try:
+ talkpage.put(talktext,
+ u"Robot: Category-redirect notification on protected page",
+ minorEdit=False) # NEEDS LOCALIZATION
+ wikipedia.output(
+ u"Left protected page notification on %s"
+ % talkpage.aslink())
+ except wikipedia.PageNotSaved:
+ wikipedia.output(
+ u"Protected page notification on %s failed"
+ % talkpage.aslink())
+
+ except wikipedia.SpamfilterError, error:
+ wikipedia.output(
+ u'Changing page %s blocked by spam filter (URL=%s)'
+ % (article.aslink(), error.url))
+ except wikipedia.NoUsername:
+ wikipedia.output(
+ u"Page %s not saved; sysop privileges required."
+ % article.aslink())
+ except wikipedia.PageNotSaved, error:
+ wikipedia.output(u"Saving page %s failed: %s"
+ % (article.aslink(), error.message))
+ return False
+
+ def move_contents(self, oldCatTitle, newCatTitle, editSummary):
+ """The worker function that moves pages out of oldCat into newCat"""
+ while True:
+ try:
+ oldCat = catlib.Category(self.site,
+ self.catprefix + oldCatTitle)
+ newCat = catlib.Category(self.site,
+ self.catprefix + newCatTitle)
+
+ # Move articles
+ found, moved = 0, 0
+ for result in self.query_results(list="categorymembers",
+ cmtitle=oldCat.title(),
+ cmprop="title|sortkey",
+ cmlimit="max"):
+ found += len(result['categorymembers'])
+ for item in result['categorymembers']:
+ article = wikipedia.Page(self.site, item['title'])
+ changed = self.change_category(article, oldCat, newCat,
+ comment=editSummary)
+ if changed: moved += 1
+
+ # pass 2: look for template doc pages
+ for result in self.query_results(list="categorymembers",
+ cmtitle=oldCat.title(),
+ cmprop="title|sortkey",
+ cmnamespace="10",
+ cmlimit="max"):
+ for item in result['categorymembers']:
+ doc = wikipedia.Page(self.site, item['title']+"/doc")
+ try:
+ old_text = doc.get()
+ except wikipedia.Error:
+ continue
+ changed = self.change_category(article, oldCat, newCat,
+ comment=editSummary)
+ if changed: moved += 1
+
+ if found:
+ wikipedia.output(u"%s: %s found, %s moved"
+ % (oldCat.title(), found, moved))
+ #Dummy edit to refresh the page, shouldn't show up in any logs.
+ try:
+ oldCat.put(oldCat.get())
+ except:
+ self.log_text.append(u'* Dummy edit at %s failed'
+ % oldCat.aslink(textlink=True))
+ self.result_queue.put((oldCatTitle, found, moved))
+ return
+ except wikipedia.ServerError:
+ wikipedia.output(u"Server error: retrying in 5 seconds...")
+ time.sleep(5)
+ continue
+ except:
+ self.result_queue.put((oldCatTitle, None, None))
+ raise
+
+ def readyToEdit(self, cat):
+ """Return True if cat not edited during cooldown period, else False."""
+ dateformat ="%Y%m%d%H%M%S"
+ today = datetime.now()
+ deadline = today + timedelta(days=-self.cooldown)
+ if cat.editTime() is None:
+ raise RuntimeError
+ return (deadline.strftime(dateformat) > cat.editTime())
+
+ def query_results(self, **data):
+ """Iterate results from API action=query, using data as parameters."""
+ addr = self.site.apipath()
+ querydata = {'action': 'query',
+ 'format': 'json',
+ 'maxlag': str(wikipedia.config.maxlag)}
+ querydata.update(data)
+ if not querydata.has_key("action")\
+ or not querydata['action'] == 'query':
+ raise ValueError(
+ "query_results: 'action' set to value other than 'query'"
+ )
+ waited = 0
+ while True:
+ response, data = self.site.postForm(addr, querydata)
+ if data.startswith(u"unknown_action"):
+ e = {'code': data[:14], 'info': data[16:]}
+ raise APIError(e)
+ try:
+ result = simplejson.loads(data)
+ except ValueError:
+ # if the result isn't valid JSON, there must be a server
+ # problem. Wait a few seconds and try again
+ # TODO: warn user; if the server is down, this could
+ # cause an infinite loop
+ wikipedia.output("Invalid API response received; retrying...")
+ time.sleep(5)
+ continue
+ if type(result) is dict and result.has_key("error"):
+ if result['error']['code'] == "maxlag":
+ print "Pausing due to server lag.\r",
+ time.sleep(5)
+ waited += 5
+ if waited % 30 == 0:
+ wikipedia.output(
+ u"(Waited %i seconds due to server lag.)"
+ % waited)
+ continue
+ else:
+ # raise error
+ raise APIError(result['error'])
+ waited = 0
+ if type(result) is list:
+ # query returned no results
+ return
+ assert type(result) is dict, \
+ "Unexpected result of type '%s' received." % type(result)
+ assert result.has_key("query"), \
+ "No 'query' response found, result keys = %s" % result.keys()
+ yield result['query']
+ if result.has_key("query-continue"):
+ assert len(result['query-continue'].keys()) == 1, \
+ "More than one query-continue key returned: %s" \
+ % result['query-continue'].keys()
+ query_type = result['query-continue'].keys()[0]
+ assert (query_type in querydata.keys()
+ or query_type in querydata.values()), \
+ "Site returned unknown query-continue type '%s'"\
+ % query_type
+ querydata.update(result['query-continue'][query_type])
+ else:
+ return
+
+ def run(self):
+ """Run the bot"""
+ self.site = wikipedia.getSite()
+ self.catprefix = self.site.namespace(14)+":"
+ self.result_queue = Queue.Queue()
+ self.log_text = []
+
+ user = self.site.loggedInAs()
+ redirect_magicwords = ["redirect"]
+ other_words = self.site.redirect()
+ if other_words:
+ redirect_magicwords.extend(other_words)
+
+ problems = []
+
+ problem_page = wikipedia.Page(self.site,
+ u"User:%(user)s/category redirect problems" % locals())
+ l = time.localtime()
+ today = "%04d-%02d-%02d" % l[:3]
+ log_page = wikipedia.Page(self.site,
+ u"User:%(user)s/category redirect logs/%(today)s"
+ % locals())
+
+ datafile = wikipedia.config.datafilepath(
+ "%s-catmovebot-data" % self.site.dbName())
+ try:
+ inp = open(datafile, "rb")
+ record = cPickle.load(inp)
+ inp.close()
+ except IOError:
+ record = {}
+ if record:
+ cPickle.dump(record, open(datafile + ".bak", "wb"))
+
+ # Set up regexes for later scanning
+ template_list = self.redir_templates[self.site.family.name
+ ][self.site.lang]
+ # regex to match soft category redirects
+ template_regexes = [
+ re.compile(
+ur"{{\s*(?:%(prefix)s\s*:\s*)?%(template)s\s*\|(\s*%(catns)s\s*:\s*)?([^}]+)}}"
+ % {'prefix': self.site.namespace(10).lower(),
+ 'template': item.replace(" ", "[ _]+"),
+ 'catns': self.site.namespace(14)},
+ re.I)
+ for item in template_list
+ ]
+ template_regex = re.compile(
+ur"{{\s*(?:%(prefix)s\s*:\s*)?(?:%(template)s)\s*\|(\s*%(catns)s\s*:\s*)?([^}]+)}}"
+ % {'prefix': self.site.namespace(10).lower(),
+ 'template': "|".join(item.replace(" ", "[ _]+")
+ for item in template_list),
+ 'catns': self.site.namespace(14)},
+ re.I)
+ # regex to match hard redirects to category pages
+ catredir_regex = re.compile(
+ur'\s*#(?:%(redir)s)\s*:?\s*\[\[\s*:?%(catns)s\s*:(.*?)\]\]\s*'
+ % {'redir': "|".join(redirect_magicwords),
+ 'catns': self.site.namespace(14)},
+ re.I)
+ # regex to match all other hard redirects
+ redir_regex = re.compile(ur"(?i)\s*#(?:%s)\s*:?\s*\[\[(.*?)\]\]"
+ % "|".join(redirect_magicwords),
+ re.I)
+
+ # check for hard-redirected categories that are not already marked
+ # with an appropriate template
+ comment = wikipedia.translate(self.site.lang, self.redir_comment)
+ for result in self.query_results(list='allpages',
+ apnamespace='14', # Category:
+ apfrom='!',
+ apfilterredir='redirects',
+ aplimit='max'):
+ gen = (wikipedia.Page(self.site, page_item['title'])
+ for page_item in result['allpages'])
+ for page in pagegenerators.PreloadingGenerator(gen, 120):
+ text = page.get(get_redirect=True)
+ if re.search(template_regex, text):
+ # this is already a soft-redirect, so skip it (for now)
+ continue
+ m = catredir_regex.match(text)
+ if m:
+ # this is a hard-redirect to a category page
+ newtext = (u"{{%(template)s|%(cat)s}}"
+ % {'cat': m.group(1),
+ 'template': template_list[0]})
+ try:
+ page.put(newtext, comment, minorEdit=True)
+ self.log_text.append(u"* Added {{tl|%s}} to %s"
+ % (template_list[0],
+ page.aslink(textlink=True)))
+ except wikipedia.Error, e:
+ self.log_text.append(
+ u"* Failed to add {{tl|%s}} to %s (%s)"
+ % (template_list[0],
+ page.aslink(textlink=True),
+ e))
+ else:
+ r = redir_regex.match(text)
+ if r:
+ problems.append(
+ u"# %s is a hard redirect to [[:%s]]"
+ % (page.aslink(textlink=True),
+ r.group(1)))
+ else:
+ problems.append(
+ u"# %s is a hard redirect; unable to extract target."
+ % page.aslink(textlink=True))
+
+ wikipedia.output("Done checking hard-redirect category pages.")
+
+ comment = wikipedia.translate(self.site.lang, self.move_comment)
+ scan_data = {
+ u'action': 'query',
+ u'list': 'embeddedin',
+ u'einamespace': '14', # Category:
+ u'eilimit': 'max',
+ u'format': 'json'
+ }
+ counts = {}
+ destmap = {}
+ catmap = {}
+ catlist = []
+ catpages = []
+ if self.often:
+ target = self.often_redirect_cat[self.site.family.name
+ ][self.site.lang]
+ else:
+ target = self.cat_redirect_cat[self.site.family.name
+ ][self.site.lang]
+
+ # get and preload all members of the category-redirect category
+ for result in self.query_results(generator=u'categorymembers',
+ gcmtitle=target,
+ gcmnamespace=u'14', # CATEGORY
+ gcmlimit=u'max',
+ prop='info'):
+ for catdata in result['pages'].values():
+ catpages.append(wikipedia.Page(self.site, catdata['title']))
+
+ wikipedia.output(u"")
+ wikipedia.output(u"Preloading %s category redirect pages"
+ % len(catpages))
+ thread_limit = int(math.log(len(catpages), 2))
+ for cat in pagegenerators.PreloadingGenerator(catpages, 120):
+ cat_title = cat.titleWithoutNamespace()
+ if "Wikipedia category redirect" in cat_title:
+# self.log_text.append("* Ignoring %s%s" % (self.catprefix, cat_title))
+ continue
+ try:
+ text = cat.get(get_redirect=True)
+ except wikipedia.Error:
+ self.log_text.append(u"* Could not load %s%s; ignoring"
+ % (self.catprefix, cat_title))
+ continue
+ match = template_regex.search(text)
+ if match is None:
+ self.log_text.append(u"* False positive: %s" % cat_title)
+ continue
+ catlist.append(cat)
+ destination = match.group(2)
+ target = catlib.Category(self.site, self.catprefix+destination)
+ destmap.setdefault(target, []).append(cat)
+ catmap[cat] = destination
+ if match.group(1):
+ # category redirect target starts with "Category:" - fix it
+ text = text[ :match.start(1)] + text[match.end(1): ]
+ cat.put(text,
+ u"Robot: fixing category redirect parameter format")
+ self.log_text.append(
+ u"* Removed category prefix from parameter in %s"
+ % cat.aslink(textlink=True))
+
+ wikipedia.output(u"")
+ wikipedia.output(u"Checking %s destination categories" % len(destmap))
+ for dest in pagegenerators.PreloadingGenerator(destmap.keys(), 120):
+ if not dest.exists():
+ for d in destmap[dest]:
+ problems.append("# %s redirects to %s"
+ % (d.aslink(textlink=True),
+ dest.aslink(textlink=True)))
+ catlist.remove(d)
+ # do a null edit on d to make it appear in the
+ # "needs repair" category (if this wiki has one)
+ try:
+ d.put(d.get(get_redirect=True))
+ except:
+ pass
+ if dest in catlist:
+ for d in destmap[dest]:
+ # is catmap[dest] also a redirect?
+ newcat = catlib.Category(self.site,
+ self.catprefix+catmap[dest])
+ while newcat in catlist:
+ if newcat == d or newcat == dest:
+ self.log_text.append(u"* Redirect loop from %s"
+ % newcat.aslink(textlink=True))
+ break
+ newcat = catlib.Category(self.site,
+ self.catprefix+catmap[newcat])
+ else:
+ self.log_text.append(
+ u"* Fixed double-redirect: %s -> %s -> %s"
+ % (d.aslink(textlink=True),
+ dest.aslink(textlink=True),
+ newcat.aslink(textlink=True)))
+ oldtext = d.get(get_redirect=True)
+ # remove the old redirect from the old text,
+ # leaving behind any non-redirect text
+ oldtext = template_regex.sub("", oldtext)
+ newtext = (u"{{category redirect|%(ncat)s}}"
+ % {'ncat': newcat.titleWithoutNamespace()})
+ newtext = newtext + oldtext.strip()
+ try:
+ d.put(newtext,
+ wikipedia.translate(self.site.lang,
+ self.dbl_redir_comment),
+ minorEdit=True)
+ except wikipedia.Error, e:
+ self.log_text.append("** Failed: %s" % str(e))
+
+ wikipedia.output(u"")
+ wikipedia.output(u"Moving pages out of %s redirected categories."
+ % len(catlist))
+ threadpool = ThreadList(limit=thread_limit)
+
+ for cat in catlist:
+ cat_title = cat.titleWithoutNamespace()
+ if not self.readyToEdit(cat):
+ counts[cat_title] = None
+ continue
+ threadpool.append(threading.Thread(target=self.move_contents,
+ args=(cat_title, catmap[cat]),
+ kwargs=dict(editSummary=comment)))
+ while len(counts) < len(catlist):
+ title, found, moved = self.result_queue.get()
+ if found is None:
+ self.log_text.append(
+ u"* [[:%s%s]]: error in move_contents thread"
+ % (self.catprefix, title))
+ else:
+ if found:
+ self.log_text.append(
+ u"* [[:%s%s]]: %d found, %d moved"
+ % (self.catprefix, title, found, moved))
+ counts[title] = found
+
+ if not self.often:
+ for cat in record.keys():
+ if cat not in counts.keys():
+ del record[cat]
+ for cat in counts.keys():
+ if counts[cat] is not None:
+ if counts[cat]:
+ record.setdefault(cat, {})[today] = counts[cat]
+ else:
+ record.setdefault(cat, {})
+ cPickle.dump(record, open(datafile, "wb"))
+
+ wikipedia.setAction(wikipedia.translate(self.site.lang,
+ self.maint_comment))
+ log_page.put("\n".join(self.log_text))
+ problem_page.put("\n".join(problems))
+
+
+def main(*args):
+ try:
+ a = wikipedia.handleArgs(*args)
+ if "-often" in a:
+ a.remove("-often")
+ often = True
+ else:
+ often = False
+ if len(a) == 1:
+ raise RuntimeError('Unrecognized argument "%s"' % a[0])
+ elif a:
+ raise RuntimeError('Unrecognized arguments: ' +
+ " ".join(('"%s"' % arg) for arg in a))
+ bot = CategoryRedirectBot(often)
+ bot.run()
+ finally:
+ wikipedia.stopme()
+
+
+if __name__ == "__main__":
+ main()