Revision: 6159
Author: russblau
Date: 2008-12-17 19:57:39 +0000 (Wed, 17 Dec 2008)
Log Message:
-----------
Improve output and logfile formatting.
Modified Paths:
--------------
branches/rewrite/pywikibot/__init__.py
branches/rewrite/pywikibot/bot.py
branches/rewrite/pywikibot/comms/http.py
branches/rewrite/pywikibot/config2.py
branches/rewrite/pywikibot/data/api.py
branches/rewrite/pywikibot/page.py
branches/rewrite/pywikibot/pagegenerators.py
branches/rewrite/pywikibot/scripts/touch.py
branches/rewrite/pywikibot/throttle.py
Modified: branches/rewrite/pywikibot/__init__.py
===================================================================
--- branches/rewrite/pywikibot/__init__.py 2008-12-17 16:11:20 UTC (rev 6158)
+++ branches/rewrite/pywikibot/__init__.py 2008-12-17 19:57:39 UTC (rev 6159)
@@ -105,7 +105,13 @@
# User interface functions (kept extremely simple for debugging)
def output(text, toStdout=False):
- print text.encode(config.console_encoding, "xmlcharrefreplace")
+ if toStdout:
+ level = STDOUT
+ else:
+ level = logging.INFO
+ logging.getLogger().log(level,
+ text.encode(config.console_encoding,
+ "xmlcharrefreplace"))
def input(prompt, password=False):
if isinstance(prompt, unicode):
@@ -169,7 +175,7 @@
# only need one drop() call because all throttles use the same global pid
try:
_sites[_sites.keys()[0]].throttle.drop()
- logger.log("VERBOSE", "Dropped throttle(s).")
+ logger.log(pywikibot.VERBOSE, "Dropped throttle(s).")
except IndexError:
pass
Modified: branches/rewrite/pywikibot/bot.py
===================================================================
--- branches/rewrite/pywikibot/bot.py 2008-12-17 16:11:20 UTC (rev 6158)
+++ branches/rewrite/pywikibot/bot.py 2008-12-17 19:57:39 UTC (rev 6159)
@@ -21,6 +21,13 @@
from pywikibot import config2 as config
+# logging levels
+
+STDOUT = 16
+VERBOSE = 18
+INPUT = 25
+
+
def calledModuleName():
"""Return the name of the module calling this function.
@@ -35,6 +42,18 @@
return os.path.basename(called)
+class LevelFilter(logging.Filter):
+ """Filter that only passes records at a specific level."""
+ def __init__(self, level=None):
+ self.level = level
+
+ def filter(self, record):
+ if self.level:
+ return record.levelno == self.level
+ else:
+ return True
+
+
def _decodeArg(arg):
if sys.platform=='win32':
if config.console_encoding == 'cp850':
@@ -77,7 +96,7 @@
for arg in args:
arg = _decodeArg(arg)
if arg == '-help':
- showHelp(moduleName)
+ showHelp()
sys.exit(0)
elif arg.startswith('-family:'):
config.family = arg[8:]
@@ -100,10 +119,8 @@
elif arg == "-debug":
if moduleName not in config.log:
config.log.append(moduleName)
- config.log.debug_log = True
+ config.debug_log = True
elif arg == '-verbose' or arg == "-v":
- pywikibot.output(u'Pywikipediabot %s' % (version.getversion()))
- pywikibot.output(u'Python %s' % (sys.version))
config.verbose_output += 1
elif arg == '-daemonize':
import daemonize
@@ -118,38 +135,60 @@
# initialize logging system for terminal-based bots
- logging.addLevelName(18, "VERBOSE") # for messages to be displayed on
- # terminal at "verbose" setting
- # use INFO for messages to be displayed
- # even on non-verbose setting
- logging.addLevelName(24, "STDOUT") # for messages to be displayed to stdout
- logging.addLevelName(26, "INPUT") # for prompts requiring user response
+ logging.addLevelName(VERBOSE, "VERBOSE")
+ # for messages to be displayed on terminal at "verbose" setting
+ # use INFO for messages to be displayed even on non-verbose setting
+ logging.addLevelName(STDOUT, "STDOUT")
+ # for messages to be displayed to stdout
+ logging.addLevelName(INPUT, "INPUT")
+ # for prompts requiring user response
- logging.basicConfig() # initializes root logger
+ logging.basicConfig(format="%(message)s") # initialize root logger
root_logger = logging.getLogger()
+ default_handler = root_logger.handlers[0]
+ root_logger.setLevel(logging.DEBUG) # all records go to logger
+ # handlers filter separately by level
if config.verbose_output:
- root_logger.setLevel("VERBOSE")
+ default_handler.setLevel(VERBOSE)
else:
- root_logger.setLevel(logging.INFO)
- if moduleName in config.log:
+ default_handler.setLevel(logging.INFO)
+ if moduleName in config.log or '*' in config.log:
if config.logfilename:
logfile = config.datafilepath(config.logfilename)
else:
- logfile = config.datafilepath("%s.log" % moduleName)
+ logfile = config.datafilepath("%s-bot.log" % moduleName)
file_handler = logging.handlers.RotatingFileHandler(
filename=logfile, maxBytes=2 << 20, backupCount=5)
if config.debug_log:
file_handler.setLevel(logging.DEBUG)
else:
- file_handler.setLevel("VERBOSE")
- logging.addHandler(file_handler)
+ file_handler.setLevel(VERBOSE)
+ form = logging.Formatter(
+ fmt="%(asctime)s %(filename)-18s:%(lineno)-4d "
+ "%(levelname)-8s %(message)s",
+ datefmt="%Y-%m-%d %H:%M:%S"
+ )
+ file_handler.setFormatter(form)
+ root_logger.addHandler(file_handler)
+ output_handler = logging.StreamHandler(strm=sys.stdout)
+ output_handler.setLevel(STDOUT)
+ output_handler.addFilter(LevelFilter(STDOUT))
+ root_logger.addHandler(output_handler)
+
+ if config.verbose_output:
+ import re
+ ver = pywikibot.__version__ # probably can be improved on
+ m = re.search(r"\$Id: .* (\d+ \d+-\d+-\d+ \d+:\d+:\d+Z) .*\$", ver)
+ pywikibot.output(u'Pywikipediabot r%s' % m.group(1))
+ pywikibot.output(u'Python %s' % sys.version)
+
return nonGlobalArgs
def showHelp(name=""):
# argument, if given, is ignored
- module = calledModuleName()
+ modname = calledModuleName()
globalHelp =u'''\
Global arguments available for all bots:
@@ -170,28 +209,30 @@
-help Shows this help text.
--log Enable the logfile. Logs will be stored in the logs
- subdirectory.
+-log Enable the logfile, using the default filename
+ '%s-bot.log'
--log:xyz Enable the logfile, using xyz as the filename.
+-log:xyz Enable the logfile, using 'xyz' as the filename.
-nolog Disable the logfile (if it is enabled by default).
+-debug Enable the logfile and include extensive debugging data.
+
-putthrottle:n Set the minimum time (in seconds) the bot will wait between
-pt:n saving pages.
-verbose Have the bot provide additional output that may be useful in
-v debugging.
-'''
+''' % modname
try:
- exec('import %s as module' % module)
+ exec('import %s as module' % modname)
helpText = module.__doc__.decode('utf-8')
if hasattr(module, 'docuReplacements'):
for key, value in module.docuReplacements.iteritems():
helpText = helpText.replace(key, value.strip('\n\r'))
pywikibot.output(helpText)
except:
- if module:
- pywikibot.output(u'Sorry, no help available for %s' % module)
+ if modname:
+ pywikibot.output(u'Sorry, no help available for %s' % modname)
logging.exception('showHelp:')
pywikibot.output(globalHelp)
Modified: branches/rewrite/pywikibot/comms/http.py
===================================================================
--- branches/rewrite/pywikibot/comms/http.py 2008-12-17 16:11:20 UTC (rev 6158)
+++ branches/rewrite/pywikibot/comms/http.py 2008-12-17 19:57:39 UTC (rev 6159)
@@ -27,7 +27,7 @@
import logging
import atexit
-from pywikibot import config
+from pywikibot import config, VERBOSE
import cookielib
import threadedhttp
@@ -57,7 +57,7 @@
# Build up HttpProcessors
-logger.info('Starting %(numthreads)i threads...' % locals())
+logger.log(VERBOSE, 'Starting %(numthreads)i threads...', locals())
for i in range(numthreads):
proc = threadedhttp.HttpProcessor(http_queue, cookie_jar, connection_pool)
proc.setDaemon(True)
@@ -68,7 +68,7 @@
def _flush():
for i in threads:
http_queue.put(None)
- logger.info('Waiting for threads to finish... ')
+ logger.log(VERBOSE, 'Waiting for threads to finish... ')
for i in threads:
i.join()
logger.debug('All threads finished.')
Modified: branches/rewrite/pywikibot/config2.py
===================================================================
--- branches/rewrite/pywikibot/config2.py 2008-12-17 16:11:20 UTC (rev 6158)
+++ branches/rewrite/pywikibot/config2.py 2008-12-17 19:57:39 UTC (rev 6159)
@@ -214,10 +214,13 @@
# log = []
# Per default, logging of interwiki.py is enabled because its logfiles can
# be used to generate so-called warnfiles.
-# This setting can be overridden by the -log or -nolog command-line arguments.
log = ['interwiki']
-logfilename = None # defaults to modulename.log
+# filename defaults to modulename-bot.log
+logfilename = None
+# set to 1 (or higher) to generate "informative" messages to terminal
verbose_output = 0
+# if True, include a lot of debugging info in logfile
+# (overrides log setting above)
debug_log = False
############## INTERWIKI SETTINGS ##############
Modified: branches/rewrite/pywikibot/data/api.py
===================================================================
--- branches/rewrite/pywikibot/data/api.py 2008-12-17 16:11:20 UTC (rev 6158)
+++ branches/rewrite/pywikibot/data/api.py 2008-12-17 19:57:39 UTC (rev 6159)
@@ -195,7 +195,7 @@
rawdata = http.request(self.site, uri)
except Exception, e: #TODO: what exceptions can occur here?
logger.warning(traceback.format_exc())
- print uri, params
+ logger.warning("%s, %s", uri, params)
self.wait()
continue
if not isinstance(rawdata, unicode):
@@ -621,7 +621,7 @@
from pywikibot import Site
logger.setLevel(pywikibot.logging.DEBUG)
mysite = Site("en", "wikipedia")
- print "starting test...."
+ pywikibot.output("starting test....")
def _test():
import doctest
doctest.testmod()
Modified: branches/rewrite/pywikibot/page.py
===================================================================
--- branches/rewrite/pywikibot/page.py 2008-12-17 16:11:20 UTC (rev 6158)
+++ branches/rewrite/pywikibot/page.py 2008-12-17 19:57:39 UTC (rev 6159)
@@ -37,8 +37,8 @@
"""
- @deprecate_arg("insite", None)
- @deprecate_arg("defaultNamespace", None)
+# @deprecate_arg("insite", None)
+# @deprecate_arg("defaultNamespace", None)
def __init__(self, source, title=u"", ns=0):
"""Instantiate a Page object.
Modified: branches/rewrite/pywikibot/pagegenerators.py
===================================================================
--- branches/rewrite/pywikibot/pagegenerators.py 2008-12-17 16:11:20 UTC (rev 6158)
+++ branches/rewrite/pywikibot/pagegenerators.py 2008-12-17 19:57:39 UTC (rev 6159)
@@ -328,92 +328,6 @@
return gen
-class ThreadedGenerator(threading.Thread):
- """Look-ahead generator class.
-
- Runs a generator in a separate thread and queues the results; can
- be called like a regular generator.
-
- Subclasses should override self.generator, _not_ self.run
-
- Important: the generator thread will stop itself if the generator's
- internal queue is exhausted; but, if the calling program does not use
- all the generated values, it must call the generator's stop() method to
- stop the background thread. Example usage:
-
- >>> gen = ThreadedGenerator(target=foo)
- >>> try:
- ... for data in gen:
- ... do_work(data)
- ... finally:
- ... gen.stop()
-
- """ #NOT CURRENTLY USED: Intended for future development
-
- def __init__(self, group=None, target=None, name="GeneratorThread",
- args=(), kwargs=None, qsize=65536):
- """Constructor. Takes same keyword arguments as threading.Thread.
-
- target must be a generator function (or other callable that returns
- an iterable object).
-
- @param qsize: The size of the lookahead queue. The larger the qsize,
- the more values will be computed in advance of use (which can eat
- up memory and processor time).
- @type qsize: int
-
- """
- if kwargs is None:
- kwargs = {}
- if target:
- self.generator = target
- if not hasattr(self, "generator"):
- raise RuntimeError("No generator for ThreadedGenerator to run.")
- self.args, self.kwargs = args, kwargs
- threading.Thread.__init__(self, group=group, name=name)
- self.queue = Queue.Queue(qsize)
- self.finished = threading.Event()
-
- def __iter__(self):
- """Iterate results from the queue."""
- if not self.isAlive() and not self.finished.isSet():
- self.start()
- # if there is an item in the queue, yield it, otherwise wait
- while not self.finished.isSet():
- try:
- yield self.queue.get(True, 0.25)
- except Queue.Empty:
- pass
- except KeyboardInterrupt:
- self.stop()
-
- def stop(self):
- """Stop the background thread."""
-## if not self.finished.isSet():
-## pywikibot.output("DEBUG: signalling %s to stop." % self)
- self.finished.set()
-
- def run(self):
- """Run the generator and store the results on the queue."""
- self.__gen = self.generator(*self.args, **self.kwargs)
- for result in self.__gen:
- while True:
- if self.finished.isSet():
-## pywikibot.output("DEBUG: %s received stop signal." % self)
- return
- try:
- self.queue.put_nowait(result)
- except Queue.Full:
- time.sleep(0.25)
- continue
- break
- # wait for queue to be emptied, then kill the thread
- while not self.finished.isSet() and not self.queue.empty():
- time.sleep(0.25)
- self.stop()
-## pywikibot.output("DEBUG: %s stopped because generator exhausted." % self)
-
-
def AllpagesPageGenerator(start ='!', namespace=None, includeredirects=True,
site=None):
"""
@@ -761,6 +675,8 @@
for page in site.search(query, number=number, namespaces = namespaces):
yield page[0]
+# following classes just ported from version 1 without revision; not tested
+
class YahooSearchPageGenerator:
'''
To use this generator, install pYsearch
@@ -785,7 +701,8 @@
def __iter__(self):
# restrict query to local site
localQuery = '%s site:%s' % (self.query, self.site.hostname())
- base = 'http://%s%s' % (self.site.hostname(), self.site.nice_get_address(''))
+ base = 'http://%s%s' % (self.site.hostname(),
+ self.site.nice_get_address(''))
for url in self.queryYahoo(localQuery):
if url[:len(base)] == base:
title = url[len(base):]
@@ -826,7 +743,8 @@
google.LICENSE_KEY = config.google_key
offset = 0
estimatedTotalResultsCount = None
- while not estimatedTotalResultsCount or offset < estimatedTotalResultsCount:
+ while not estimatedTotalResultsCount \
+ or offset < estimatedTotalResultsCount:
while (True):
# Google often yields 502 errors.
try:
@@ -887,7 +805,8 @@
def __iter__(self):
# restrict query to local site
localQuery = '%s site:%s' % (self.query, self.site.hostname())
- base = 'http://%s%s' % (self.site.hostname(), self.site.nice_get_address(''))
+ base = 'http://%s%s' % (self.site.hostname(),
+ self.site.nice_get_address(''))
for url in self.queryGoogle(localQuery):
if url[:len(base)] == base:
title = url[len(base):]
Modified: branches/rewrite/pywikibot/scripts/touch.py
===================================================================
--- branches/rewrite/pywikibot/scripts/touch.py 2008-12-17 16:11:20 UTC (rev 6158)
+++ branches/rewrite/pywikibot/scripts/touch.py 2008-12-17 19:57:39 UTC (rev 6159)
@@ -39,11 +39,12 @@
text = page.get(get_redirect = self.touch_redirects)
page.save("Pywikibot touch script")
except pywikibot.NoPage:
- print "Page %s does not exist?!" % page.aslink()
+ pywikibot.output("Page %s does not exist?!" % page.aslink())
except pywikibot.IsRedirectPage:
- print "Page %s is a redirect; skipping." % page.aslink()
+ pywikibot.output("Page %s is a redirect; skipping."
+ % page.aslink())
except pywikibot.LockedPage:
- print "Page %s is locked?!" % page.aslink()
+ pywikibot.output("Page %s is locked?!" % page.aslink())
def main(*args):
Modified: branches/rewrite/pywikibot/throttle.py
===================================================================
--- branches/rewrite/pywikibot/throttle.py 2008-12-17 16:11:20 UTC (rev 6158)
+++ branches/rewrite/pywikibot/throttle.py 2008-12-17 19:57:39 UTC (rev 6159)
@@ -39,7 +39,7 @@
multiplydelay=True, verbosedelay=False):
self.lock = threading.RLock()
self.mysite = str(site)
- self.logfn = config.datafilepath('throttle.log')
+ self.ctrlfilename = config.datafilepath('throttle.ctrl')
self.mindelay = mindelay
if self.mindelay is None:
self.mindelay = config.minthrottle
@@ -73,7 +73,7 @@
count = 1
# open throttle.log
try:
- f = open(self.logfn, 'r')
+ f = open(self.ctrlfilename, 'r')
except IOError:
if not pid:
pass
@@ -110,7 +110,7 @@
processes.append({'pid': pid,
'time': self.checktime,
'site': mysite})
- f = open(self.logfn, 'w')
+ f = open(self.ctrlfilename, 'w')
processes.sort(key=lambda p:(p['pid'], p['site']))
for p in processes:
f.write("%(pid)s %(time)s %(site)s\n" % p)
@@ -187,7 +187,7 @@
self.checktime = 0
processes = []
try:
- f = open(self.logfn, 'r')
+ f = open(self.ctrlfilename, 'r')
except IOError:
return
else:
@@ -206,15 +206,14 @@
processes.append({'pid': this_pid,
'time': ptime,
'site': this_site})
- f = open(self.logfn, 'w')
+ f = open(self.ctrlfilename, 'w')
processes.sort(key=lambda p:p['pid'])
for p in processes:
f.write("%(pid)s %(time)s %(site)s\n" % p)
f.close()
def __call__(self, requestsize=1, write=False):
- """
- Block the calling program if the throttle time has not expired.
+ """Block the calling program if the throttle time has not expired.
Parameter requestsize is the number of Pages to be read/written;
multiply delay time by an appropriate factor.
@@ -248,8 +247,7 @@
self.lock.release()
def lag(self, lagtime):
- """
- Seize the throttle lock due to server lag.
+ """Seize the throttle lock due to server lag.
This will prevent any thread from accessing this site.
Bugs item #2422802, was opened at 2008-12-12 23:31
Message generated for change (Settings changed) made by a_engels
You can respond by visiting:
https://sourceforge.net/tracker/?func=detail&atid=603138&aid=2422802&group_…
Please note that this message will contain a full copy of the comment thread,
including the initial issue submission, for this request,
not just the latest update.
Category: interwiki
Group: None
>Status: Closed
>Resolution: Fixed
Priority: 5
Private: No
Submitted By: Julian Mendez (julianmendez)
Assigned to: Nobody/Anonymous (nobody)
Summary: not every page with w:fr:Template:Sigle is a disambiguation
Initial Comment:
Template:Sigle (in English, acronym) at fr.wiki may be used for common articles (non-disambiguation pages).
Some examples are http://fr.wikipedia.org/wiki/COBOL , and http://fr.wikipedia.org/wiki/BASIC .
Please, see the file 'families/wikipedia_family.py', where 'Sigle' is connected to disambiguation pages.
Pywikipedia [http] trunk/pywikipedia (r6144, Dec 12 2008, 14:27:52)
Python 2.4.4 (#1, Jan 10 2007, 01:25:01) [C]
----------------------------------------------------------------------
Comment By: Andre Engels (a_engels)
Date: 2008-12-17 07:32
Message:
Ok, has been removed.
----------------------------------------------------------------------
You can respond by visiting:
https://sourceforge.net/tracker/?func=detail&atid=603138&aid=2422802&group_…
Bugs item #2422802, was opened at 2008-12-12 23:31
Message generated for change (Comment added) made by a_engels
You can respond by visiting:
https://sourceforge.net/tracker/?func=detail&atid=603138&aid=2422802&group_…
Please note that this message will contain a full copy of the comment thread,
including the initial issue submission, for this request,
not just the latest update.
Category: interwiki
Group: None
Status: Open
Resolution: None
Priority: 5
Private: No
Submitted By: Julian Mendez (julianmendez)
Assigned to: Nobody/Anonymous (nobody)
Summary: not every page with w:fr:Template:Sigle is a disambiguation
Initial Comment:
Template:Sigle (in English, acronym) at fr.wiki may be used for common articles (non-disambiguation pages).
Some examples are http://fr.wikipedia.org/wiki/COBOL , and http://fr.wikipedia.org/wiki/BASIC .
Please, see the file 'families/wikipedia_family.py', where 'Sigle' is connected to disambiguation pages.
Pywikipedia [http] trunk/pywikipedia (r6144, Dec 12 2008, 14:27:52)
Python 2.4.4 (#1, Jan 10 2007, 01:25:01) [C]
----------------------------------------------------------------------
>Comment By: Andre Engels (a_engels)
Date: 2008-12-17 07:32
Message:
Ok, has been removed.
----------------------------------------------------------------------
You can respond by visiting:
https://sourceforge.net/tracker/?func=detail&atid=603138&aid=2422802&group_…
Revision: 6156
Author: russblau
Date: 2008-12-16 19:40:20 +0000 (Tue, 16 Dec 2008)
Log Message:
-----------
update properties
Modified Paths:
--------------
branches/rewrite/pywikibot/catlib.py
branches/rewrite/pywikibot/exceptions.py
branches/rewrite/pywikibot/page.py
branches/rewrite/pywikibot/pagegenerators.py
branches/rewrite/pywikibot/site.py
branches/rewrite/pywikibot/textlib.py
branches/rewrite/pywikibot/throttle.py
branches/rewrite/pywikibot/tools.py
Property Changed:
----------------
branches/rewrite/pywikibot/__init__.py
branches/rewrite/pywikibot/bot.py
branches/rewrite/pywikibot/catlib.py
branches/rewrite/pywikibot/exceptions.py
branches/rewrite/pywikibot/page.py
branches/rewrite/pywikibot/pagegenerators.py
branches/rewrite/pywikibot/site.py
branches/rewrite/pywikibot/textlib.py
branches/rewrite/pywikibot/throttle.py
branches/rewrite/pywikibot/tools.py
Property changes on: branches/rewrite/pywikibot/__init__.py
___________________________________________________________________
Added: svn:keywords
+ Author Date Id Revision
Property changes on: branches/rewrite/pywikibot/bot.py
___________________________________________________________________
Added: svn:keywords
+ Author Date Id Revision
Modified: branches/rewrite/pywikibot/catlib.py
===================================================================
--- branches/rewrite/pywikibot/catlib.py 2008-12-16 19:34:48 UTC (rev 6155)
+++ branches/rewrite/pywikibot/catlib.py 2008-12-16 19:40:20 UTC (rev 6156)
@@ -1,22 +1,22 @@
-# -*- coding: utf-8 -*-
-"""
-WARNING: THIS MODULE EXISTS SOLELY TO PROVIDE BACKWARDS-COMPATIBILITY.
-
-Do not use in new scripts; use the source to find the appropriate
-function/method instead.
-
-"""
-#
-# (C) Pywikipedia bot team, 2008
-#
-# Distributed under the terms of the MIT license.
-#
-__version__ = '$Id: $'
-
-
-from pywikibot import Category
-
-
-def change_category(article, oldCat, newCat, comment=None, sortKey=None,
- inPlace=True):
- return article.change_category(oldCat, newCat, comment, sortKey, inPlace)
+# -*- coding: utf-8 -*-
+"""
+WARNING: THIS MODULE EXISTS SOLELY TO PROVIDE BACKWARDS-COMPATIBILITY.
+
+Do not use in new scripts; use the source to find the appropriate
+function/method instead.
+
+"""
+#
+# (C) Pywikipedia bot team, 2008
+#
+# Distributed under the terms of the MIT license.
+#
+__version__ = '$Id$'
+
+
+from pywikibot import Category
+
+
+def change_category(article, oldCat, newCat, comment=None, sortKey=None,
+ inPlace=True):
+ return article.change_category(oldCat, newCat, comment, sortKey, inPlace)
Property changes on: branches/rewrite/pywikibot/catlib.py
___________________________________________________________________
Added: svn:keywords
+ Author Date Id Revision
Added: svn:eol-style
+ native
Modified: branches/rewrite/pywikibot/exceptions.py
===================================================================
--- branches/rewrite/pywikibot/exceptions.py 2008-12-16 19:34:48 UTC (rev 6155)
+++ branches/rewrite/pywikibot/exceptions.py 2008-12-16 19:40:20 UTC (rev 6156)
@@ -1,87 +1,87 @@
-# -*- coding: utf-8 -*-
-"""
-Exception classes used throughout the framework.
-"""
-#
-# (C) Pywikipedia bot team, 2008
-#
-# Distributed under the terms of the MIT license.
-#
-__version__ = '$Id: $'
-
-
-import sys
-
-# TODO: These are copied from wikipedia.py; not certain that all of them
-# will be needed in the rewrite.
-
-class Error(Exception):
- """Wikipedia error"""
- def __init__(self, arg):
- try:
- self.string = arg.encode(sys.stderr.encoding, "xmlcharrefreplace")
- except (AttributeError, TypeError):
- self.string = arg.encode("ascii", "xmlcharrefreplace")
- def __str__(self):
- return self.string
-
-class NoUsername(Error):
- """Username is not in user-config.py"""
-
-class NoPage(Error):
- """Page does not exist"""
-
-class NoSuchSite(Error):
- """Site does not exist"""
-
-class IsRedirectPage(Error):
- """Page is a redirect page"""
-
-class IsNotRedirectPage(Error):
- """Page is not a redirect page"""
-
-class CircularRedirect(Error):
- """Page is a circular redirect
-
- Exception argument is the redirect target; this may be the same title
- as this page or a different title (in which case the target page directly
- or indirectly redirects back to this one)
-
- """
-
-class LockedPage(Error):
- """Page is locked"""
-
-class SectionError(Error):
- """The section specified by # does not exist"""
-
-class PageNotSaved(Error):
- """Saving the page has failed"""
-
-class EditConflict(PageNotSaved):
- """There has been an edit conflict while uploading the page"""
-
-class SpamfilterError(PageNotSaved):
- """Saving the page has failed because the MediaWiki spam filter detected a blacklisted URL."""
- def __init__(self, arg):
- self.url = arg
- self.args = arg,
-
-class ServerError(Error):
- """Got unexpected server response"""
-
-class BadTitle(Error):
- """Server responded with BadTitle."""
-
-# UserBlocked exceptions should in general not be caught. If the bot has
-# been blocked, the bot operator should address the reason for the block
-# before continuing.
-class UserBlocked(Error):
- """Your username or IP has been blocked"""
-
-class PageNotFound(Error):
- """Page not found in list"""
-
-class CaptchaError(Error):
- """Captcha is asked and config.solve_captcha == False."""
-
+# -*- coding: utf-8 -*-
+"""
+Exception classes used throughout the framework.
+"""
+#
+# (C) Pywikipedia bot team, 2008
+#
+# Distributed under the terms of the MIT license.
+#
+__version__ = '$Id$'
+
+
+import sys
+
+# TODO: These are copied from wikipedia.py; not certain that all of them
+# will be needed in the rewrite.
+
+class Error(Exception):
+ """Wikipedia error"""
+ def __init__(self, arg):
+ try:
+ self.string = arg.encode(sys.stderr.encoding, "xmlcharrefreplace")
+ except (AttributeError, TypeError):
+ self.string = arg.encode("ascii", "xmlcharrefreplace")
+ def __str__(self):
+ return self.string
+
+class NoUsername(Error):
+ """Username is not in user-config.py"""
+
+class NoPage(Error):
+ """Page does not exist"""
+
+class NoSuchSite(Error):
+ """Site does not exist"""
+
+class IsRedirectPage(Error):
+ """Page is a redirect page"""
+
+class IsNotRedirectPage(Error):
+ """Page is not a redirect page"""
+
+class CircularRedirect(Error):
+ """Page is a circular redirect
+
+ Exception argument is the redirect target; this may be the same title
+ as this page or a different title (in which case the target page directly
+ or indirectly redirects back to this one)
+
+ """
+
+class LockedPage(Error):
+ """Page is locked"""
+
+class SectionError(Error):
+ """The section specified by # does not exist"""
+
+class PageNotSaved(Error):
+ """Saving the page has failed"""
+
+class EditConflict(PageNotSaved):
+ """There has been an edit conflict while uploading the page"""
+
+class SpamfilterError(PageNotSaved):
+ """Saving the page has failed because the MediaWiki spam filter detected a blacklisted URL."""
+ def __init__(self, arg):
+ self.url = arg
+ self.args = arg,
+
+class ServerError(Error):
+ """Got unexpected server response"""
+
+class BadTitle(Error):
+ """Server responded with BadTitle."""
+
+# UserBlocked exceptions should in general not be caught. If the bot has
+# been blocked, the bot operator should address the reason for the block
+# before continuing.
+class UserBlocked(Error):
+ """Your username or IP has been blocked"""
+
+class PageNotFound(Error):
+ """Page not found in list"""
+
+class CaptchaError(Error):
+ """Captcha is asked and config.solve_captcha == False."""
+
Property changes on: branches/rewrite/pywikibot/exceptions.py
___________________________________________________________________
Added: svn:keywords
+ Author Date Id Revision
Added: svn:eol-style
+ native
Modified: branches/rewrite/pywikibot/page.py
===================================================================
--- branches/rewrite/pywikibot/page.py 2008-12-16 19:34:48 UTC (rev 6155)
+++ branches/rewrite/pywikibot/page.py 2008-12-16 19:40:20 UTC (rev 6156)
@@ -1,1886 +1,1886 @@
-# -*- coding: utf-8 -*-
-"""
-Objects representing various types of MediaWiki pages.
-"""
-#
-# (C) Pywikipedia bot team, 2008
-#
-# Distributed under the terms of the MIT license.
-#
-__version__ = '$Id: $'
-
-import pywikibot
-from pywikibot import deprecate_arg
-from pywikibot import config
-import pywikibot.site
-import pywikibot.textlib
-
-import htmlentitydefs
-import logging
-import re
-import sys
-import threading
-import unicodedata
-import urllib
-
-logger = logging.getLogger("wiki")
-
-reNamespace = re.compile("^(.+?) *: *(.*)$")
-
-
-class Page(object):
- """Page: A MediaWiki page
-
- This object only implements internally methods that do not require
- reading from or writing to the wiki. All other methods are delegated
- to the Site object.
-
- """
-
- @deprecate_arg("insite", None)
- @deprecate_arg("defaultNamespace", None)
- def __init__(self, source, title=u"", ns=0):
- """Instantiate a Page object.
-
- Three calling formats are supported:
-
- - If the first argument is a Page, create a copy of that object.
- This can be used to convert an existing Page into a subclass
- object, such as Category or ImagePage. (If the title is also
- given as the second argument, creates a copy with that title;
- this is used when pages are moved.)
- - If the first argument is a Site, create a Page on that Site
- using the second argument as the title (may include a section),
- and the third as the namespace number. The namespace number is
- mandatory, even if the title includes the namespace prefix. This
- is the preferred syntax when using an already-normalized title
- obtained from api.php or a database dump. WARNING: may produce
- invalid objects if page title isn't in normal form!
- - If the first argument is a Link, create a Page from that link.
- This is the preferred syntax when using a title scraped from
- wikitext, URLs, or another non-normalized source.
-
- @param source: the source of the page
- @type source: Link, Page (or subclass), or Site
- @param title: normalized title of the page; required if source is a
- Site, ignored otherwise
- @type title: unicode
- @param ns: namespace number; required if source is a Site, ignored
- otherwise
- @type ns: int
-
- """
- if isinstance(source, pywikibot.site.BaseSite):
- self._site = source
- if ns not in source.namespaces():
- raise pywikibot.Error(
- "Invalid namespace '%i' for site %s."
- % (ns, source.sitename()))
- self._ns = ns
- if ns and not title.startswith(source.namespace(ns)+u":"):
- title = source.namespace(ns) + u":" + title
- elif not ns and u":" in title:
- pos = title.index(u':')
- nsindex = source.ns_index(title[ :pos])
- if nsindex:
- self._ns = nsindex
- if u"#" in title:
- title, self._section = title.split(u"#", 1)
- else:
- self._section = None
- if not title:
- raise pywikibot.Error(
- "Page object cannot be created from Site without title.")
- self._title = title
- elif isinstance(source, Page):
- # copy all of source's attributes to this object
- self.__dict__ = source.__dict__
- if title:
- # overwrite title
- if ":" in title:
- prefix = title[ :title.index(":")]
- self._ns = site.ns_index(prefix)
- if self._ns is None:
- self._ns = 0
- else:
- title = title[title.index(":")+1 : ].strip(" _")
- self._title = "%s:%s" % (
- self.site().namespace(self._ns),
- self._title)
- else:
- self._ns = 0
- if "#" in title:
- self._section = title[title.index("#") + 1 : ].strip(" _")
- title = title[ : title.index("#")].strip(" _")
- self._title = title
- elif isinstance(source, Link):
- self._site = source.site
- self._section = source.section
- self._ns = source.namespace
- self._title = source.title
- # reassemble the canonical title from components
- if self._ns:
- self._title = "%s:%s" % (self.site().namespace(self._ns),
- self._title)
- else:
- raise pywikibot.Error(
- "Invalid argument type '%s' in Page constructor: %s"
- % (type(source), source))
- if self._section is not None:
- self._title = self._title + "#" + self._section
- self._revisions = {}
-
- def site(self):
- """Return the Site object for the wiki on which this Page resides."""
- return self._site
-
- def namespace(self):
- """Return the number of the namespace of the page."""
- return self._ns
-
- @deprecate_arg("decode", None)
- @deprecate_arg("savetitle", "asUrl")
- def title(self, underscore=False, savetitle=False, withNamespace=True,
- withSection=True, asUrl=False, asLink=False,
- allowInterwiki=True, forceInterwiki=False, textlink=False,
- as_filename=False):
- """Return the title of this Page, as a Unicode string.
-
- @param underscore: if true, replace all ' ' characters with '_'
- @param withNamespace: if false, omit the namespace prefix
- @param withSection: if false, omit the section
- @param asUrl: if true, quote title as if in an URL
- @param asLink: if true, return the title in the form of a wikilink
- @param allowInterwiki: (only used if asLink is true) if true, format
- the link as an interwiki link if necessary
- @param forceInterwiki: (only used if asLink is true) if true, always
- format the link as an interwiki link
- @param textlink: (only used if asLink is true) if true, place a ':'
- before Category: and Image: links
- @param as_filename: if true, replace any characters that are unsafe
- in filenames
-
- """
- title = self._title
- if not withNamespace and self._ns != 0:
- title = title.split(u':', 1)[1]
- if not withSection and self._section:
- title = title.split(u'#', 1)[0]
- if underscore or asUrl:
- title = title.replace(u' ', u'_')
- if asUrl:
- encodedTitle = title.encode(self.site().encoding())
- title = urllib.quote(encodedTitle)
- if asLink:
- if forceInterwiki or (allowInterwiki and
- (self.site().family.name != config.family
- or self.site().code != config.mylang)):
- if self.site().family.name != config.family \
- and self.site().family.name != self.site().code:
- return u'[[%s:%s:%s]]' % (self.site().family.name,
- self.site().code,
- self._title)
- else:
- # use this form for sites like commons, where the
- # code is the same as the family name
- return u'[[%s:%s]]' % (self.site().code,
- self._title)
- elif textlink and (self.isImage() or self.isCategory()):
- return u'[[:%s]]' % title
- else:
- return u'[[%s]]' % title
- if as_filename:
- # Replace characters that are not possible in file names on some
- # systems.
- # Spaces are possible on most systems, but are bad for URLs.
- for forbidden in ':*?/\\ ':
- title = title.replace(forbidden, '_')
- return title
-
- @deprecate_arg("decode", None)
- @deprecate_arg("underscore", None)
- def section(self):
- """Return the name of the section this Page refers to.
-
- The section is the part of the title following a '#' character, if
- any. If no section is present, return None.
-
- """
- if self._section:
- return self._section
- else:
- return None
-
- def __str__(self):
- """Return a console representation of the pagelink."""
- return self.title(asLink=True, forceInterwiki=True
- ).encode(sys.stderr.encoding)
-
- def __unicode__(self):
- return self.title(asLink=True, forceInterwiki=True)
-
- def __repr__(self):
- """Return a more complete string representation."""
- return u"%s(%s)" % (self.__class__.__name__,
- self.title().encode(sys.stderr.encoding))
-
- def __cmp__(self, other):
- """Test for equality and inequality of Page objects.
-
- Page objects are "equal" if and only if they are on the same site
- and have the same normalized title, including section if any.
-
- Page objects are sortable by namespace first, then by title.
-
- """
- if not isinstance(other, Page):
- # especially, return -1 if other is None
- return -1
- if not self.site() == other.site():
- return cmp(self.site(), other.site())
- if self.namespace() != other.namespace():
- return cmp(self.namespace(), other.namespace())
- owntitle = self.title(withNamespace=False)
- othertitle = other.title(withNamespace=False)
- return cmp(owntitle, othertitle)
-
- def __hash__(self):
- # Pseudo method that makes it possible to store Page objects as keys
- # in hash-tables. This relies on the fact that the string
- # representation of an instance can not change after the construction.
- return hash(unicode(self))
-
- def autoFormat(self):
- """Return L{date.autoFormat} dictName and value, if any.
-
- Value can be a year, date, etc., and dictName is 'YearBC',
- 'Year_December', or another dictionary name. Please note that two
- entries may have exactly the same autoFormat, but be in two
- different namespaces, as some sites have categories with the
- same names. Regular titles return (None, None).
-
- """
- if not hasattr(self, '_autoFormat'):
- from pywikibot import date
- self._autoFormat = date.getAutoFormat(
- self.site().code,
- self.title(withNamespace=False)
- )
- return self._autoFormat
-
- def isAutoTitle(self):
- """Return True if title of this Page is in the autoFormat dictionary."""
- return self.autoFormat()[0] is not None
-
- @deprecate_arg("throttle", None)
- @deprecate_arg("nofollow_redirects", None)
- @deprecate_arg("change_edit_time", None)
- def get(self, force=False, get_redirect=False, sysop=False):
- """Return the wiki-text of the page.
-
- This will retrieve the page from the server if it has not been
- retrieved yet, or if force is True. This can raise the following
- exceptions that should be caught by the calling code:
-
- - NoPage: The page does not exist
- - IsRedirectPage: The page is a redirect. The argument of the
- exception is the title of the page it redirects to.
- - SectionError: The section does not exist on a page with a #
- link
-
- @param force: reload all page attributes, including errors.
- @param get_redirect: return the redirect text, do not follow the
- redirect, do not raise an exception.
- @param sysop: if the user has a sysop account, use it to retrieve
- this page
-
- """
- if force:
- # When forcing, we retry the page no matter what. Old exceptions
- # do not apply any more.
- for attr in ['_redirarg', '_getexception']:
- if hasattr(self, attr):
- delattr(self,attr)
- else:
- # Make sure we re-raise an exception we got on an earlier attempt
- if hasattr(self, '_redirarg') and not get_redirect:
- raise pywikibot.IsRedirectPage, self._redirarg
- elif hasattr(self, '_getexception'):
- raise self._getexception
- if force or not hasattr(self, "_revid") \
- or not self._revid in self._revisions \
- or self._revisions[self._revid].text is None:
- self.site().loadrevisions(self, getText=True, sysop=sysop)
- # TODO: Exception handling for no-page, redirects, etc.
-
- return self._revisions[self._revid].text
-
- @deprecate_arg("throttle", None)
- @deprecate_arg("nofollow_redirects", None)
- @deprecate_arg("change_edit_time", None)
- def getOldVersion(self, oldid, force=False, get_redirect=False,
- sysop=False):
- """Return text of an old revision of this page; same options as get().
-
- @param oldid: The revid of the revision desired.
-
- """
- if force or not oldid in self._revisions \
- or self._revisions[oldid].text is None:
- self.site().loadrevisions(self, getText=True, revids=oldid,
- sysop=sysop)
- # TODO: what about redirects, errors?
- return self._revisions[oldid].text
-
- def permalink(self):
- """Return the permalink URL for current revision of this page."""
- return "%s://%s/%sindex.php?title=%s&oldid=%s" \
- % (self.site().protocol(),
- self.site().hostname(),
- self.site().scriptpath(),
- self.title(asUrl=True),
- self.latestRevision())
-
- def latestRevision(self):
- """Return the current revision id for this page."""
- if not hasattr(self, '_revid'):
- self.site().loadrevisions(self)
- return self._revid
-
- def _textgetter(self):
- """Return the current (edited) wikitext, loading it if necessary."""
- if not hasattr(self, '_text') or self._text is None:
- try:
- self._text = self.get()
- except pywikibot.NoPage:
- # TODO: what other exceptions might be returned?
- self._text = u""
- return self._text
-
- def _textsetter(self, value):
- """Update the edited wikitext"""
- self._text = unicode(value)
-
- def _cleartext(self):
- """Delete the edited wikitext"""
- if hasattr(self, "_text"):
- del self._text
-
- text = property(_textgetter, _textsetter, _cleartext,
- "The edited wikitext (unicode) of this Page")
-
- def expand_text(self):
- """Return the page text with all templates expanded."""
- req = pywikibot.data.api.Request(action="expandtemplates",
- text=self.text,
- title=self.title(withSection=False),
- site=self.site())
- result = req.submit()
- return result["expandtemplates"]["*"]
-
- def userName(self):
- """Return name or IP address of last user to edit page."""
- return self._revisions[self.latestRevision()].user
-
- def isIpEdit(self):
- """Return True if last editor was unregistered."""
- return self._revisions[self.latestRevision()].anon
-
- def editTime(self):
- """Return timestamp (in ISO 8601 format) of last revision to page."""
- return self._revisions[self.latestRevision()].timestamp
-
- def previousRevision(self):
- """Return the revision id for the previous revision of this Page."""
- vh = self.getVersionHistory(revCount=2)
- revkey = sorted(self._revisions.keys(), reverse=True)[1]
- return revkey
-
- def exists(self):
- """Return True if page exists on the wiki, even if it's a redirect.
-
- If the title includes a section, return False if this section isn't
- found.
-
- """
- return self.site().page_exists(self)
-
- def isRedirectPage(self):
- """Return True if this is a redirect, False if not or not existing."""
- return self.site().page_isredirect(self)
-
- def isEmpty(self):
- """Return True if the page text has less than 4 characters.
-
- Character count ignores language links and category links.
- Can raise the same exceptions as get().
-
- """
- txt = self.get()
- txt = pywikibot.textlib.removeLanguageLinks(txt, site = self.site())
- txt = pywikibot.textlib.removeCategoryLinks(txt, site = self.site())
- if len(txt) < 4:
- return True
- else:
- return False
-
- def isTalkPage(self):
- """Return True if this page is in any talk namespace."""
- ns = self.namespace()
- return ns >= 0 and ns % 2 == 1
-
- def toggleTalkPage(self):
- """Return other member of the article-talk page pair for this Page.
-
- If self is a talk page, returns the associated content page;
- otherwise, returns the associated talk page. The returned page need
- not actually exist on the wiki.
-
- Returns None if self is a special page.
-
- """
- ns = self.namespace()
- if ns < 0: # Special page
- return None
- if self.isTalkPage():
- if self.namespace() == 1:
- return Page(self.site(), self.title(withNamespace=False))
- else:
- return Page(self.site(),
- self.site().namespace(ns - 1) + ':'
- + self.title(withNamespace=False))
- else:
- return Page(self.site(),
- self.site().namespace(ns + 1) + ':'
- + self.title(withNamespace=False))
-
- def isCategory(self):
- """Return True if the page is a Category, False otherwise."""
- return self.namespace() == 14
-
- def isImage(self):
- """Return True if this is an image description page, False otherwise."""
- return self.namespace() == 6
-
- def isDisambig(self):
- """Return True if this is a disambiguation page, False otherwise.
-
- Relies on the presence of specific templates, identified in
- the Family file or on a wiki page, to identify disambiguation
- pages.
-
- By default, loads a list of template names from the Family file;
- if the value in the Family file is None, looks for the list on
- [[MediaWiki:Disambiguationspage]].
-
- """
- if not hasattr(self, "_isDisambig"):
- if not hasattr(self.site(), "_disambigtemplates"):
- self.site()._disambigtemplates = \
- self.site().family.disambig(self.site().code)
- if self.site()._disambigtemplates is None:
- try:
- disambigpages = Page(self.site(),
- "MediaWiki:Disambiguationspage")
- self.site()._disambigtemplates = [
- link.title(withNamespace=False)
- for link in disambigpages.linkedPages()
- if link.namespace() == 10
- ]
- except NoPage:
- self.site()._disambigtemplates = ['Disambig']
- for t in self.templates():
- if t.title(withNamespace=False) in self.site()._disambigtemplates:
- self._isDisambig = True
- break
- else:
- self._isDisambig = False
- return self._isDisambig
-
- def getReferences(self, follow_redirects=True, withTemplateInclusion=True,
- onlyTemplateInclusion=False, redirectsOnly=False,
- namespaces=None):
- """Return an iterator all pages that refer to or embed the page.
-
- If you need a full list of referring pages, use
- C{pages = list(s.getReferences())}
-
- @param follow_redirects: if True, also iterate pages that link to a
- redirect pointing to the page.
- @param withTemplateInclusion: if True, also iterate pages where self
- is used as a template.
- @param onlyTemplateInclusion: if True, only iterate pages where self
- is used as a template.
- @param redirectsOnly: if True, only iterate redirects to self.
- @param namespaces: only iterate pages in these namespaces
-
- """
- # N.B.: this method intentionally overlaps with backlinks() and
- # embeddedin(). Depending on the interface, it may be more efficient
- # to implement those methods in the site interface and then combine
- # the results for this method, or to implement this method and then
- # split up the results for the others.
- return self.site().pagereferences(
- self, follow_redirects, redirectsOnly,
- withTemplateInclusion, onlyTemplateInclusion,
- namespaces)
-
- def backlinks(self, followRedirects=True, filterRedirects=None,
- namespaces=None):
- """Return an iterator for pages that link to this page.
-
- @param followRedirects: if True, also iterate pages that link to a
- redirect pointing to the page.
- @param filterRedirects: if True, only iterate redirects; if False,
- omit redirects; if None, do not filter
- @param namespaces: only iterate pages in these namespaces
-
- """
- return self.site().pagebacklinks(self, followRedirects, filterRedirects,
- namespaces)
-
- def embeddedin(self, filter_redirects=None, namespaces=None):
- """Return an iterator for pages that embed this page as a template.
-
- @param filterRedirects: if True, only iterate redirects; if False,
- omit redirects; if None, do not filter
- @param namespaces: only iterate pages in these namespaces
-
- """
- return self.site().page_embeddedin(self, filter_redirects, namespaces)
-
- def canBeEdited(self):
- """Return bool indicating whether this page can be edited.
-
- This returns True if and only if:
- - page is unprotected, and bot has an account for this site, or
- - page is protected, and bot has a sysop account for this site.
-
- """
- return self.site().page_can_be_edited(self)
-
- def botMayEdit(self):
- """Return True if this page allows bots to edit it.
-
- This will be True if the page doesn't contain {{bots}} or
- {{nobots}}, or it contains them and the active bot is allowed to
- edit this page. (This method is only useful on those sites that
- recognize the bot-exclusion protocol; on other sites, it will always
- return True.)
-
- The framework enforces this restriction by default. It is possible
- to override this by setting ignore_bot_templates=True in
- user_config.py, or using page.put(force=True).
-
- """ # TODO: move this to Site object?
- if config.ignore_bot_templates: #Check the "master ignore switch"
- return True
- try:
- templates = self.templatesWithParams();
- except (pywikibot.NoPage,
- pywikibot.IsRedirectPage,
- pywikibot.SectionError):
- return True
- for template in templates:
- title = template[0].title(withNamespace=False)
- if title == 'Nobots':
- return False
- elif title == 'Bots':
- if len(template[1]) == 0:
- return True
- else:
- (ttype, bots) = template[1][0].split('=', 1)
- bots = bots.split(',')
- if ttype == 'allow':
- if 'all' in bots or username in bots:
- return True
- else:
- return False
- if ttype == 'deny':
- if 'all' in bots or username in bots:
- return False
- else:
- return True
- # no restricting template found
- return True
-
- def save(self, comment=None, watch=None, minor=True, force=False,
- async=False, callback=None):
- """Save the current contents of page's text to the wiki.
-
- @param comment: The edit summary for the modification (optional, but
- most wikis strongly encourage its use)
- @type comment: unicode
- @param watch: if True, add or if False, remove this Page to/from bot
- user's watchlist; if None, leave watchlist status unchanged
- @type watch: bool or None
- @param minor: if True, mark this edit as minor
- @type minor: bool
- @param force: if True, ignore botMayEdit() setting
- @type force: bool
- @param async: if True, launch a separate thread to save
- asynchronously
- @param callback: a callable object that will be called after the
- page put operation. This object must take two arguments: (1) a
- Page object, and (2) an exception instance, which will be None
- if the page was saved successfully. The callback is intended for
- use by bots that need to keep track of which saves were
- successful.
-
- """
- if not comment:
- comment = pywikibot.default_comment # needs to be defined
- if watch is None:
- unwatch = False
- watch = False
- else:
- unwatch = not watch
- if not force and not self.botMayEdit:
- raise pywikibot.PageNotSaved(
- "Page %s not saved; editing restricted by {{bots}} template"
- % self.title(asLink=True))
- if async:
- thd = threading.Thread(
- target=self._save,
- args=(comment, minor, watch, unwatch, callback)
- )
- pywikibot.threadpool.append(thd)
- thd.start()
- else:
- self._save(comment, minor, watch, unwatch, callback)
-
- def _save(self, comment, minor, watch, unwatch, callback):
- err = None
- try:
- done = self.site().editpage(self, summary=comment, minor=minor,
- watch=watch, unwatch=unwatch)
- if not done:
- logger.warn("Page %s not saved" % self.title(asLink=True))
- else:
- logger.info("Page %s saved" % self.title(asLink=True))
- except pywikibot.Error, err:
- logger.exception("Error saving page %s" % self.title(asLink=True))
- if callback:
- callback(self, err)
-
- def put(self, newtext, comment=u'', watchArticle=None, minorEdit=True,
- force=False, async=False, callback=None):
- """Save the page with the contents of the first argument as the text.
-
- This method is maintained primarily for backwards-compatibility.
- For new code, using Page.save() is preferred. See save() method
- docs for all parameters not listed here.
-
- @param newtext: The complete text of the revised page.
- @type newtext: unicode
-
- """
- self.text = newtext
- return self.save(comment, watchArticle, minorEdit, force,
- async, callback)
-
- def put_async(self, newtext, comment=u'', watchArticle=None,
- minorEdit=True, force=False, callback=None):
- """Put page on queue to be saved to wiki asynchronously.
-
- Asynchronous version of put (takes the same arguments), which places
- pages on a queue to be saved by a daemon thread. All arguments are
- the same as for .put(). This version is maintained solely for
- backwards-compatibility.
-
- """
- return self.put(self, newtext, comment, watchArticle,
- minorEdit, force, callback, async=True)
-
- def linkedPages(self):
- """Iterate Pages that this Page links to.
-
- Only returns pages from "normal" internal links. Image and category
- links are omitted unless prefixed with ":". Embedded templates are
- omitted (but links within them are returned). All interwiki and
- external links are omitted.
-
- @return: a generator that yields Page objects.
-
- """
- return self.site().pagelinks(self)
-
- def interwiki(self, expand=True):
- """Iterate interwiki links in the page text, excluding language links.
-
- @param expand: if True (default), include interwiki links found in
- templates transcluded onto this page; if False, only iterate
- interwiki links found in this page's own wikitext
- @return: a generator that yields Link objects
-
- """
- # This function does not exist in the API, so it has to be
- # implemented by screen-scraping
- if expand:
- text = self.expand_text()
- else:
- text = self.text
- for linkmatch in pywikibot.link_regex.finditer(
- pywikibot.textlib.removeDisabledParts(text)):
- linktitle = linkmatch.group("title")
- link = Link(linktitle, self.site())
- # only yield links that are to a different site and that
- # are not language links
- try:
- if link.site != self.site():
- if linktitle.lstrip().startswith(":"):
- # initial ":" indicates not a language link
- yield link
- elif link.site.family != self.site().family:
- # link to a different family is not a language link
- yield link
- except pywikibot.Error:
- # ignore any links with invalid contents
- continue
-
- def langlinks(self):
- """Iterate all interlanguage links on this page.
-
- @return: a generator that yields Link objects.
-
- """
- return self.site().pagelanglinks(self)
-
- @deprecate_arg("followRedirects", None)
- @deprecate_arg("loose", None)
- def imagelinks(self, followRedirects=None, loose=None):
- """Iterate ImagePage objects for images displayed on this Page.
-
- @return: a generator that yields ImagePage objects.
-
- """
- return self.site().pageimages(self)
-
- def templates(self):
- """Iterate Page objects for templates used on this Page.
-
- Template parameters are ignored. This method only returns embedded
- templates, not template pages that happen to be referenced through
- a normal link.
-
- """
- return self.site().pagetemplates(self)
-
- def templatesWithParams(self):
- """Iterate templates used on this Page.
-
- @return: a generator that yields a tuple for each use of a template
- in the page, with the template Page as the first entry and a list of
- parameters as the second entry.
-
- """
- templates = pywikibot.textlib.extract_templates_and_params(self.text)
- # backwards-compatibility: convert the dict returned as the second
- # element into a list in the format used by old scripts
- result = []
- for template in templates:
- args = template[1]
- positional = []
- named = {}
- for key in sorted(args.keys()):
- try:
- int(key)
- except ValueError:
- named[key] = args[key]
- else:
- positional.append(args[key])
- for name in named:
- positional.append("%s=%s" % (name, named[name]))
- result.append((pywikibot.Page(
- pywikibot.Link(template[0], self.site())),
- positional))
- return result
-
- @deprecate_arg("nofollow_redirects", None)
- def categories(self, withSortKey=False):
- """Iterate categories that the article is in.
-
- @param withSortKey: if True, include the sort key in each Category.
- @return: a generator that yields Category objects.
-
- """
- return self.site().pagecategories(self, withSortKey=withSortKey)
-
- def extlinks(self):
- """Iterate all external URLs (not interwiki links) from this page.
-
- @return: a generator that yields unicode objects containing URLs.
-
- """
- return self.site().page_extlinks(self)
-
- def getRedirectTarget(self):
- """Return a Page object for the target this Page redirects to.
-
- If this page is not a redirect page, will raise an IsNotRedirectPage
- exception. This method also can raise a NoPage exception.
-
- """
- if not self.isRedirectPage():
- raise pywikibot.IsNotRedirectPage
- if not isinstance(self._redir, Page):
- self.site().getredirtarget(self)
- return self._redir
-
- @deprecate_arg("forceReload", None)
- def getVersionHistory(self, reverseOrder=False, getAll=False,
- revCount=500):
- """Load the version history page and return history information.
-
- Return value is a list of tuples, where each tuple represents one
- edit and is built of revision id, edit date/time, user name, and
- edit summary. Starts with the most current revision, unless
- reverseOrder is True. Defaults to getting the first revCount edits,
- unless getAll is True.
-
- """
- if getAll:
- limit = None
- else:
- limit = revCount
- self.site().loadrevisions(self, getText=False, rvdir=reverseOrder,
- limit=limit)
- if getAll:
- revCount = len(self._revisions)
- return [ ( self._revisions[rev].revid,
- self._revisions[rev].timestamp,
- self._revisions[rev].user,
- self._revisions[rev].comment
- ) for rev in sorted(self._revisions.keys(),
- reverse=not reverseOrder)[ : revCount]
- ]
-
- def getVersionHistoryTable(self, forceReload=False, reverseOrder=False,
- getAll=False, revCount=500):
- """Return the version history as a wiki table."""
- result = '{| border="1"\n'
- result += '! oldid || date/time || username || edit summary\n'
- for oldid, time, username, summary \
- in self.getVersionHistory(forceReload=forceReload,
- reverseOrder=reverseOrder,
- getAll=getAll, revCount=revCount):
- result += '|----\n'
- result += '| %s || %s || %s || <nowiki>%s</nowiki>\n'\
- % (oldid, time, username, summary)
- result += '|}\n'
- return result
-
- def fullVersionHistory(self):
- """Iterate all previous versions including wikitext.
-
- @return: A generator that yields tuples consisting of revision ID,
- edit date/time, user name and content
- """
- return self.site().loadrevisions(self, withText=True)
-
- def contributingUsers(self):
- """Return a set of usernames (or IPs) of users who edited this page."""
- edits = self.getVersionHistory()
- users = set([edit[2] for edit in edits])
- return users
-
- @deprecate_arg("throttle", None)
- def move(self, newtitle, reason=None, movetalkpage=True, sysop=False,
- deleteAndMove=False, safe=True):
- """Move this page to a new title.
-
- @param newtitle: The new page title.
- @param reason: The edit summary for the move.
- @param movetalkpage: If true, move this page's talk page (if it exists)
- @param sysop: Try to move using sysop account, if available
- @param deleteAndMove: if move succeeds, delete the old page
- (usually requires sysop privileges, depending on wiki settings)
- @param safe: If false, attempt to delete existing page at newtitle
- (if there is one) and then move this page to that title
-
- """
- if reason is None:
- logger.info(u'Moving %s to [[%s]].'
- % (self.title(asLink=True), newtitle))
- reason = pywikibot.input(u'Please enter a reason for the move:')
- # TODO: implement "safe" parameter
- # TODO: implement "sysop" parameter
- return self.site().movepage(self, newtitle, reason,
- movetalk=movetalkpage,
- noredirect=deleteAndMove)
-
- @deprecate_arg("throttle", None)
- def delete(self, reason=None, prompt=True, throttle=None, mark=False):
- """Deletes the page from the wiki. Requires administrator status.
-
- @param reason: The edit summary for the deletion.
- @param prompt: If true, prompt user for confirmation before deleting.
- @param mark: if true, and user does not have sysop rights, place a
- speedy-deletion request on the page instead.
-
- """
- if reason is None:
- logger.info(u'Deleting %s.' % (self.title(asLink=True)))
- reason = pywikibot.input(u'Please enter a reason for the deletion:')
- answer = u'y'
- if prompt and not hasattr(self.site(), '_noDeletePrompt'):
- answer = pywikibot.inputChoice(u'Do you want to delete %s?'
- % self.title(asLink = True, forceInterwiki = True),
- ['Yes', 'No', 'All'],
- ['Y', 'N', 'A'],
- 'N')
- if answer in ['a', 'A']:
- answer = 'y'
- self.site()._noDeletePrompt = True
- if answer in ['y', 'Y']:
- return self.site().delete(self, reason, mark=mark)
-
- def loadDeletedRevisions(self):
- """Retrieve all deleted revisions for this Page from Special/Undelete.
-
- Stores all revisions' timestamps, dates, editors and comments in
- self._deletedRevs attribute.
-
- @return: list of timestamps (which can be used to retrieve revisions
- later on).
-
- """
- return self.site().loadDeletedRevisions(self)
-
- def getDeletedRevision(self, timestamp, retrieveText=False):
- """Return a particular deleted revision by timestamp.
-
- @return: a list of [date, editor, comment, text, restoration
- marker]. text will be None, unless retrieveText is True (or has
- been retrieved earlier). If timestamp is not found, returns
- None.
-
- """
- return self.site().getDeletedRevision(self, timestamp,
- getText=retrieveText)
-
- def markDeletedRevision(self, timestamp, undelete=True):
- """Mark the revision identified by timestamp for undeletion.
-
- @param undelete: if False, mark the revision to remain deleted.
-
- """
- if self._deletedRevs == None:
- self.loadDeletedRevisions()
- if not self._deletedRevs.has_key(timestamp):
- #TODO: Throw an exception?
- return None
- self._deletedRevs[timestamp][4] = undelete
- self._deletedRevsModified = True
-
- @deprecate_arg("throttle", None)
- def undelete(self, comment=None):
- """Undelete revisions based on the markers set by previous calls.
-
- If no calls have been made since loadDeletedRevisions(), everything
- will be restored.
-
- Simplest case::
- Page(...).undelete('This will restore all revisions')
-
- More complex::
- pg = Page(...)
- revs = pg.loadDeletedRevsions()
- for rev in revs:
- if ... #decide whether to undelete a revision
- pg.markDeletedRevision(rev) #mark for undeletion
- pg.undelete('This will restore only selected revisions.')
-
- @param comment: The undeletion edit summary.
-
- """
- if comment is None:
- logger.info(u'Preparing to undelete %s.'
- % (self.title(asLink=True)))
- comment = pywikibot.input(
- u'Please enter a reason for the undeletion:')
- return self.site().undelete(self, comment)
-
- @deprecate_arg("throttle", None)
- def protect(self, edit='sysop', move='sysop', unprotect=False,
- reason=None, prompt=True):
- """(Un)protect a wiki page. Requires administrator status.
-
- Valid protection levels (in MediaWiki 1.12) are '' (equivalent to
- 'none'), 'autoconfirmed', and 'sysop'.
-
- @param edit: Level of edit protection
- @param move: Level of move protection
- @param unprotect: If true, unprotect the page (equivalent to setting
- all protection levels to '')
- @param reason: Edit summary.
- @param prompt: If true, ask user for confirmation.
-
- """
- if reason is None:
- if unprotect:
- un = u'un'
- else:
- un = u''
- logger.info(u'Preparing to %sprotect %s.'
- % (un, self.title(asLink=True)))
- reason = pywikibot.input(u'Please enter a reason for the action:')
- if unprotect:
- edit = move = ""
- answer = 'y'
- if prompt and not hasattr(self.site(), '_noProtectPrompt'):
- answer = pywikibot.inputChoice(
- u'Do you want to change the protection level of %s?'
- % self.title(asLink=True, forceInterwiki = True),
- ['Yes', 'No', 'All'], ['Y', 'N', 'A'], 'N')
- if answer in ['a', 'A']:
- answer = 'y'
- self.site()._noProtectPrompt = True
- if answer in ['y', 'Y']:
- return self.site().protect(self, edit, move, reason)
-
- def change_category(article, oldCat, newCat, comment=None, sortKey=None,
- inPlace=True):
- """Remove page from oldCat and add it to newCat.
-
- oldCat and newCat should be Category objects.
- If newCat is None, the category will be removed.
-
- """ # TODO: document remaining arguments
- cats = self.categories(get_redirect=True)
- site = self.site()
- changesMade = False
-
- if not self.canBeEdited():
- pywikibot.output(u"Can't edit %s, skipping it..."
- % self.title(asLink=True))
- return False
- if inPlace == True:
- newtext = pywikibot.textlib.replaceCategoryInPlace(
- self.text, oldCat, newCat)
- if newtext == self.text:
- pywikibot.output(
- u'No changes in made in page %s.'
- % self.title(asLink=True))
- return False
- try:
- self.put(newtext, comment)
- return True
- except pywikibot.EditConflict:
- pywikibot.output(
- u'Skipping %s because of edit conflict'
- % self.title(asLink=True))
- except pywikibot.LockedPage:
- pywikibot.output(u'Skipping locked page %s'
- % self.title(asLink=True))
- except pywikibot.SpamfilterError, error:
- pywikibot.output(
- u'Changing page %s blocked by spam filter (URL=%s)'
- % (self.title(asLink=True), error.url))
- except pywikibot.NoUsername:
- pywikibot.output(
- u"Page %s not saved; sysop privileges required."
- % self.title(asLink=True))
- except pywikibot.PageNotSaved, error:
- pywikibot.output(u"Saving page %s failed: %s"
- % (self.title(asLink=True), error.message))
- return False
-
- # This loop will replace all occurrences of the category to be changed,
- # and remove duplicates.
- newCatList = []
- newCatSet = set()
- for i in range(len(cats)):
- cat = cats[i]
- if cat == oldCat:
- changesMade = True
- if not sortKey:
- sortKey = cat.sortKey
- if newCat:
- if newCat.title() not in newCatSet:
- newCategory = Category(site, newCat.title(),
- sortKey=sortKey)
- newCatSet.add(newCat.title())
- newCatList.append(newCategory)
- elif cat.title() not in newCatSet:
- newCatSet.add(cat.title())
- newCatList.append(cat)
-
- if not changesMade:
- pywikibot.output(u'ERROR: %s is not in category %s!'
- % (self.title(asLink=True), oldCat.title()))
- else:
- try:
- text = pywikibot.textlib.replaceCategoryLinks(self.text,
- newCatList)
- except ValueError:
- # Make sure that the only way replaceCategoryLinks() can return
- # a ValueError is in the case of interwiki links to self.
- pywikibot.output(
- u'Skipping %s because of interwiki link to self' % self)
- try:
- self.put(text, comment)
- except pywikibot.EditConflict:
- pywikibot.output(
- u'Skipping %s because of edit conflict' % self.title())
- except pywikibot.SpamfilterError, e:
- pywikibot.output(
- u'Skipping %s because of blacklist entry %s'
- % (self.title(), e.url))
- except pywikibot.LockedPage:
- pywikibot.output(
- u'Skipping %s because page is locked' % self.title())
- except pywikibot.PageNotSaved, error:
- pywikibot.output(u"Saving page %s failed: %s"
- % (self.title(asLink=True), error.message))
-
-######## DEPRECATED METHODS ########
-
- def encoding(self):
- """DEPRECATED: use Site.encoding() instead"""
- logger.debug(u"Page.encoding() is deprecated; use Site.encoding().")
- return self.site().encoding()
-
- def titleWithoutNamespace(self, underscore=False):
- """DEPRECATED: use self.title(withNamespace=False) instead."""
- logger.debug(
- u"Page.titleWithoutNamespace() method is deprecated.")
- return self.title(underscore=underscore, withNamespace=False,
- withSection=False)
-
- def titleForFilename(self):
- """DEPRECATED: use self.title(as_filename=True) instead."""
- logger.debug(
- u"Page.titleForFilename() method is deprecated.")
- return self.title(as_filename=True)
-
- def sectionFreeTitle(self, underscore=False):
- """DEPRECATED: use self.title(withSection=False) instead."""
- logger.debug(
- u"Page.sectionFreeTitle() method is deprecated.")
- return self.title(underscore=underscore, withSection=False)
-
- def aslink(self, forceInterwiki=False, textlink=False, noInterwiki=False):
- """DEPRECATED: use self.title(asLink=True) instead."""
- logger.debug(u"Page.aslink() method is deprecated.")
- return self.title(asLink=True, forceInterwiki=forceInterwiki,
- allowInterwiki=not noInterwiki, textlink=textlink)
-
- def urlname(self):
- """Return the Page title encoded for use in an URL.
-
- DEPRECATED: use self.title(asUrl=True) instead.
-
- """
- logger.debug(u"Page.urlname() method is deprecated.")
- return self.title(asUrl=True)
-
-####### DISABLED METHODS (warnings provided) ######
- # these methods are easily replaced by editing the page's text using
- # textlib methods and then using put() on the result.
-
- def removeImage(self, image, put=False, summary=None, safe=True):
- """Old method to remove all instances of an image from page."""
- logger.warning(u"Page.removeImage() is no longer supported.")
-
- def replaceImage(self, image, replacement=None, put=False, summary=None,
- safe=True):
- """Old method to replace all instances of an image with another."""
- logger.warning(u"Page.replaceImage() is no longer supported.")
-
-
-class ImagePage(Page):
- """A subclass of Page representing an image descriptor wiki page.
-
- Supports the same interface as Page, with the following added methods:
-
- getImagePageHtml : Download image page and return raw HTML text.
- fileURL : Return the URL for the image described on this
- page.
- fileIsOnCommons : Return True if image stored on Wikimedia
- Commons.
- fileIsShared : Return True if image stored on Wikitravel
- shared repository.
- getFileMd5Sum : Return image file's MD5 checksum.
- getFileVersionHistory : Return the image file's version history.
- getFileVersionHistoryTable: Return the version history in the form of a
- wiki table.
- usingPages : Iterate Pages on which the image is displayed.
-
- """
- def __init__(self, source, title=u"", insite=None):
- Page.__init__(self, source, title, 6)
- if self.namespace() != 6:
- raise ValueError(u"'%s' is not in the image namespace!" % title)
-
- def getImagePageHtml(self):
- """
- Download the image page, and return the HTML, as a unicode string.
-
- Caches the HTML code, so that if you run this method twice on the
- same ImagePage object, the page will only be downloaded once.
- """
- if not hasattr(self, '_imagePageHtml'):
- from pywikibot.data import http
- path = "%s/index.php?title=%s" \
- % (self.site().scriptpath(), self.title(asUrl=True))
- self._imagePageHtml = http.request(self.site(), path)
- return self._imagePageHtml
-
- def fileUrl(self):
- """Return the URL for the image described on this page."""
- # TODO add scaling option?
- if not hasattr(self, '_imageinfo'):
- self._imageinfo = self.site().getimageinfo(self) #FIXME
- return self._imageinfo['url']
-
- def fileIsOnCommons(self):
- """Return True if the image is stored on Wikimedia Commons"""
- return self.fileUrl().startswith(
- 'http://upload.wikimedia.org/wikipedia/commons/')
-
- def fileIsShared(self):
- """Return True if image is stored on any known shared repository."""
- # as of now, the only known repositories are commons and wikitravel
- if 'wikitravel_shared' in self.site().shared_image_repository():
- return self.fileUrl().startswith(
- u'http://wikitravel.org/upload/shared/')
- return self.fileIsOnCommons()
-
- def getFileMd5Sum(self):
- """Return image file's MD5 checksum."""
- logger.debug(
- "ImagePage.getFileMd5Sum() is deprecated; use getFileSHA1Sum().")
-# FIXME: MD5 might be performed on incomplete file due to server disconnection
-# (see bug #1795683).
- import md5, urllib
- f = urllib.urlopen(self.fileUrl())
- # TODO: check whether this needs a User-Agent header added
- md5Checksum = md5.new(f.read()).hexdigest()
- f.close()
- return md5Checksum
-
- def getFileSHA1Sum(self):
- """Return image file's SHA1 checksum."""
- if not hasattr(self, '_imageinfo'):
- self._imageinfo = self.site().getimageinfo(self) #FIXME
- return self._imageinfo['sha1']
-
- def getFileVersionHistory(self):
- """Return the image file's version history.
-
- @return: An iterator yielding tuples containing (timestamp,
- username, resolution, filesize, comment).
-
- """
- #TODO; return value may need to change
- return self.site().getimageinfo(self, history=True) #FIXME
-
- def getFileVersionHistoryTable(self):
- """Return the version history in the form of a wiki table."""
- lines = []
- #TODO: if getFileVersionHistory changes, make sure this follows it
- for (datetime, username, resolution, size, comment) \
- in self.getFileVersionHistory():
- lines.append('| %s || %s || %s || %s || <nowiki>%s</nowiki>' \
- % (datetime, username, resolution, size, comment))
- return u'{| border="1"\n! date/time || username || resolution || size || edit summary\n|----\n' + u'\n|----\n'.join(lines) + '\n|}'
-
- def usingPages(self):
- """Yield Pages on which the image is displayed."""
- return self.site().getimageusage(self)
-
-
-class Category(Page):
- """A page in the Category: namespace"""
-
- @deprecate_arg("sortKey", None)
- def __init__(self, source, title=u"", insite=None):
- """All parameters are the same as for Page() constructor.
-
- """
- Page.__init__(self, source, title, 14)
- if self.namespace() != 14:
- raise ValueError(u"'%s' is not in the category namespace!"
- % title)
-
- @deprecate_arg("forceInterwiki", None)
- @deprecate_arg("textlink", None)
- @deprecate_arg("noInterwiki", None)
- def aslink(self, sortKey=u''):
- """Return a link to place a page in this Category.
-
- Use this only to generate a "true" category link, not for interwikis
- or text links to category pages.
-
- @param sortKey: The sort key for the article to be placed in this
- Category; if omitted, default sort key is used.
- @type sortKey: (optional) unicode
-
- """
- if sortKey:
- titleWithSortKey = '%s|%s' % (self.title(withSection=False),
- self.sortKey)
- else:
- titleWithSortKey = self.title(withSection=False)
- return '[[%s]]' % titleWithSortKey
-
- @deprecate_arg("startFrom", None)
- @deprecate_arg("cacheResults", None)
- def subcategories(self, recurse=False):
- """Iterate all subcategories of the current category.
-
- @param recurse: if not False or 0, also iterate subcategories of
- subcategories. If an int, limit recursion to this number of
- levels. (Example: recurse=1 will iterate direct subcats and
- first-level sub-sub-cats, but no deeper.)
- @type recurse: int or bool
-
- """
- if not isinstance(recurse, bool) and recurse:
- recurse = recurse - 1
- if not hasattr(self, "_subcats"):
- self._subcats = []
- for member in self.site().categorymembers(self, namespaces=[14]):
- subcat = Category(self.site(), member.title())
- self._subcats.append(subcat)
- yield subcat
- if recurse:
- for item in subcat.subcategories(recurse):
- yield item
- else:
- for subcat in self._subcats:
- yield subcat
- if recurse:
- for item in subcat.subcategories(recurse):
- yield item
-
- @deprecate_arg("startFrom", None)
- def articles(self, recurse=False):
- """
- Yields all articles in the current category.
-
- @param recurse: if not False or 0, also iterate articles in
- subcategories. If an int, limit recursion to this number of
- levels. (Example: recurse=1 will iterate articles in first-level
- subcats, but no deeper.)
- @type recurse: int or bool
-
- """
- namespaces = [x for x in self.site().namespaces().keys()
- if x>=0 and x!=14]
- for member in self.site().categorymembers(self,
- namespaces=namespaces):
- yield member
- if recurse:
- if not isinstance(recurse, bool) and recurse:
- recurse = recurse - 1
- for subcat in self.subcategories():
- for article in subcat.articles(recurse):
- yield article
-
- def isEmptyCategory(self):
- """Return True if category has no members (including subcategories)."""
- for member in self.site().categorymembers(self, limit=1):
- return False
- return True
-
- def copyTo(self, catname):
- """
- Copy text of category page to a new page. Does not move contents.
-
- @param catname: New category title (without namespace)
- @return: True if copying was successful, False if target page
- already existed.
-
- """
- # This seems far too specialized to be in the top-level framework
- catname = self.site().category_namespace() + ':' + catname
- targetCat = Category(self.site(), catname)
- if targetCat.exists():
- logger.warn('Target page %s already exists!'
- % targetCat.title())
- return False
- else:
- logger.info('Moving text from %s to %s.'
- % (self.title(), targetCat.title()))
- authors = ', '.join(self.contributingUsers())
- creationSummary = pywikibot.translate(
- self.site(), msg_created_for_renaming
- ) % (self.title(), authors)
- targetCat.put(self.get(), creationSummary)
- return True
-
- def copyAndKeep(self, catname, cfdTemplates):
- """Copy partial category page text (not contents) to a new title.
-
- Like copyTo above, except this removes a list of templates (like
- deletion templates) that appear in the old category text. It also
- removes all text between the two HTML comments BEGIN CFD TEMPLATE
- and END CFD TEMPLATE. (This is to deal with CFD templates that are
- substituted.)
-
- Returns true if copying was successful, false if target page already
- existed.
-
- @param catname: New category title (without namespace)
- @param cfdTemplates: A list (or iterator) of templates to be removed
- from the page text
- @return: True if copying was successful, False if target page
- already existed.
-
- """
- # I don't see why we need this as part of the framework either
- catname = self.site().category_namespace() + ':' + catname
- targetCat = Category(self.site(), catname)
- if targetCat.exists():
- logger.warn('Target page %s already exists!'
- % targetCat.title())
- return False
- else:
- logger.info('Moving text from %s to %s.'
- % (self.title(), targetCat.title()))
- authors = ', '.join(self.contributingUsers())
- creationSummary = pywikibot.translate(
- self.site(), msg_created_for_renaming
- ) % (self.title(), authors)
- newtext = self.get()
- for regexName in cfdTemplates:
- matchcfd = re.compile(r"{{%s.*?}}" % regexName, re.IGNORECASE)
- newtext = matchcfd.sub('',newtext)
- matchcomment = re.compile(
- r"<!--BEGIN CFD TEMPLATE-->.*?<!--END CFD TEMPLATE-->",
- re.IGNORECASE | re.MULTILINE | re.DOTALL)
- newtext = matchcomment.sub('', newtext)
- pos = 0
- while (newtext[pos:pos+1] == "\n"):
- pos = pos + 1
- newtext = newtext[pos:]
- targetCat.put(newtext, creationSummary)
- return True
-
-#### DEPRECATED METHODS ####
- def subcategoriesList(self, recurse=False):
- """DEPRECATED: Equivalent to list(self.subcategories(...))"""
- logger.debug("Category.subcategoriesList() method is deprecated.")
- return sorted(list(set(self.subcategories(recurse))))
-
- def articlesList(self, recurse=False):
- """DEPRECATED: equivalent to list(self.articles(...))"""
- logger.debug("Category.articlesList() method is deprecated.")
- return sorted(list(set(self.articles(recurse))))
-
- def supercategories(self):
- """DEPRECATED: equivalent to self.categories()"""
- logger.debug("Category.supercategories() method is deprecated.")
- return self.categories()
-
- def supercategoriesList(self):
- """DEPRECATED: equivalent to list(self.categories(...))"""
- logger.debug("Category.articlesList() method is deprecated.")
- return sorted(list(set(self.categories())))
-
-
-class Revision(object):
- """A structure holding information about a single revision of a Page."""
- def __init__(self, revid, timestamp, user, anon=False, comment=u"",
- text=None, minor=False):
- """All parameters correspond to object attributes (e.g., revid
- parameter is stored as self.revid)
-
- @param revid: Revision id number
- @type revid: int
- @param text: Revision wikitext.
- @type text: unicode, or None if text not yet retrieved
- @param timestamp: Revision time stamp (in ISO 8601 format)
- @type timestamp: unicode
- @param user: user who edited this revision
- @type user: unicode
- @param anon: user is unregistered
- @type anon: bool
- @param comment: edit comment text
- @type comment: unicode
- @param minor: edit flagged as minor
- @type minor: bool
-
- """
- self.revid = revid
- self.text = text
- self.timestamp = timestamp
- self.user = user
- self.anon = anon
- self.comment = comment
- self.minor = minor
-
-
-class Link(object):
- """A Mediawiki link (local or interwiki)
-
- Has the following attributes:
-
- - site: The Site object for the wiki linked to
- - namespace: The namespace of the page linked to (int)
- - title: The title of the page linked to (unicode); does not include
- namespace or section
- - section: The section of the page linked to (unicode or None); this
- contains any text following a '#' character in the title
- - anchor: The anchor text (unicode or None); this contains any text
- following a '|' character inside the link
-
- """
- illegal_titles_pattern = re.compile(
- # Matching titles will be held as illegal.
- u'''[^ %!\"$&'()*,\\-.\\/0-9:;=?@A-Z\\\\^_`a-z~\u0080-\uFFFF+]'''
- # URL percent encoding sequences interfere with the ability
- # to round-trip titles -- you can't link to them consistently.
- u'|%[0-9A-Fa-f]{2}'
- # XML/HTML character references produce similar issues.
- u'|&[A-Za-z0-9\x80-\xff]+;'
- u'|&#[0-9]+;'
- u'|&#x[0-9A-Fa-f]+;'
- )
-
- def __init__(self, text, source=None, defaultNamespace=0):
- """Constructor
-
- @param text: the link text (everything appearing between [[ and ]]
- on a wiki page)
- @type text: unicode
- @param source: the Site on which the link was found (not necessarily
- the site to which the link refers)
- @type source: Site
- @param defaultNamespace: a namespace to use if the link does not
- contain one (defaults to 0)
- @type defaultNamespace: int
-
- """
- self._text = text
- self._source = source
- self._defaultns = defaultNamespace
-
- def parse(self):
- """Parse text; called internally when accessing attributes"""
-
- # First remove the anchor, which is stored unchanged, if there is one
- if u"|" in self._text:
- self._text, self._anchor = self._text.split(u"|", 1)
- else:
- self._anchor = None
-
- if self._source is None:
- self._source = pywikibot.Site()
- self._site = self._source
-
- # Clean up the name, it can come from anywhere.
- # Convert HTML entities to unicode
- t = html2unicode(self._text)
-
- # Convert URL-encoded characters to unicode
- t = url2unicode(t, site=self._site)
-
- # Normalize unicode string to a NFC (composed) format to allow proper
- # string comparisons. According to
- # http://svn.wikimedia.org/viewvc/mediawiki/branches/REL1_6/phase3/includes/n…
- # the mediawiki code normalizes everything to NFC, not NFKC (which
- # might result in information loss).
- t = unicodedata.normalize('NFC', t)
-
- # This code was adapted from Title.php : secureAndSplit()
- #
- if u'\ufffd' in t:
- raise pywikibot.Error("Title contains illegal char (\\uFFFD)")
- self._namespace = self._defaultns
-
- # Replace underscores by spaces
- t = t.replace(u"_", u" ")
- # replace multiple spaces and underscores with a single space
- while u" " in t: t = t.replace(u" ", u" ")
- # Strip spaces at both ends
- t = t.strip(" ")
- # Remove left-to-right and right-to-left markers.
- t = t.replace(u"\u200e", u"").replace(u"\u200f", u"")
-
- firstPass = True
- while u":" in t:
- # Initial colon indicates main namespace rather than default
- if t.startswith(u":"):
- self._namespace = 0
- # remove the colon but continue processing
- # remove any subsequent whitespace
- t = t.lstrip(u":").lstrip(u" ")
- continue
-
- fam = self._site.family
- prefix = t[ :t.index(u":")].lower()
- ns = self._site.ns_index(prefix)
- if ns:
- # Ordinary namespace
- t = t[t.index(u":"): ].lstrip(u":").lstrip(u" ")
- self._namespace = ns
- break
- if prefix in fam.langs.keys()\
- or prefix in fam.get_known_families(site=self._site):
- # looks like an interwiki link
- if not firstPass:
- # Can't make a local interwiki link to an interwiki link.
- raise pywikibot.Error(
- "Improperly formatted interwiki link '%s'"
- % self._text)
- t = t[t.index(u":"): ].lstrip(u":").lstrip(u" ")
- if prefix in fam.langs.keys():
- newsite = pywikibot.Site(prefix, fam)
- else:
- otherlang = self._site.code
- familyName = fam.get_known_families(site=self._site)[prefix]
- if familyName in ['commons', 'meta']:
- otherlang = familyName
- try:
- newsite = pywikibot.Site(otherlang, familyName)
- except ValueError:
- raise pywikibot.Error("""\
-%s is not a local page on %s, and the %s family is
-not supported by PyWikiBot!"""
- % (title, self._site(), familyName))
-
- # Redundant interwiki prefix to the local wiki
- if newsite == self._site:
- if not t:
- # Can't have an empty self-link
- raise pywikibot.Error(
- "Invalid link title: '%s'" % self._text)
- firstPass = False
- continue
- self._site = newsite
- else:
- break # text before : doesn't match any known prefix
-
- if u"#" in t:
- t, sec = t.split(u'#', 1)
- t, self._section = t.rstrip(), sec.lstrip()
- else:
- self._section = None
-
- # Reject illegal characters.
- m = Link.illegal_titles_pattern.search(t)
- if m:
- raise pywikibot.Error(
- u"Invalid title: contains illegal char(s) '%s'" % m.group(0))
-
- # Pages with "/./" or "/../" appearing in the URLs will
- # often be unreachable due to the way web browsers deal
- #* with 'relative' URLs. Forbid them explicitly.
-
- if u'.' in t and (
- t == u'.' or t == u'..'
- or t.startswith(u"./")
- or t.startswith(u"../")
- or u"/./" in t
- or u"/../" in t
- or t.endswith(u"/.")
- or t.endswith(u"/..")
- ):
- raise pywikibot.Error(
- "Invalid title (contains . / combinations): '%s'"
- % self._text)
-
- # Magic tilde sequences? Nu-uh!
- if u"~~~" in t:
- raise pywikibot.Error("Invalid title (contains ~~~): '%s'" % self._text)
-
- if self._namespace != -1 and len(t) > 255:
- raise pywikibot.Error("Invalid title (over 255 bytes): '%s'" % t)
-
- if self._site.case() == 'first-letter':
- t = t[:1].upper() + t[1:]
-
- # Can't make a link to a namespace alone...
- # "empty" local links can only be self-links
- # with a fragment identifier.
- if not t and self._site == self._source and self._namespace != 0:
- raise ValueError("Invalid link (no page title): '%s'" % self._text)
-
- self._title = t
-
- # define attributes, to be evaluated lazily
-
- @property
- def site(self):
- if not hasattr(self, "_site"):
- self.parse()
- return self._site
-
- @property
- def namespace(self):
- if not hasattr(self, "_namespace"):
- self.parse()
- return self._namespace
-
- @property
- def title(self):
- if not hasattr(self, "_title"):
- self.parse()
- return self._title
-
- @property
- def section(self):
- if not hasattr(self, "_section"):
- self.parse()
- return self._section
-
- @property
- def anchor(self):
- if not hasattr(self, "_anchor"):
- self.parse()
- return self._anchor
-
- def astext(self, onsite=None):
- """Return a text representation of the link.
-
- @param onsite: if specified, present as a (possibly interwiki) link
- from the given site; otherwise, present as an internal link on
- the source site.
-
- """
- if onsite is None:
- onsite = self.site
- title = self.title
- if self.namespace:
- title = onsite.namespace(self.namespace) + ":" + title
- if self.section:
- title = title + "#" + self.section
- if onsite == self.site:
- return u'[[%s]]' % title
- if onsite.family == self.site.family:
- return u'[[%s:%s]]' % (self.site.code, title)
- if self.site.family.name == self.site.code:
- # use this form for sites like commons, where the
- # code is the same as the family name
- return u'[[%s:%s]]' % (self.site.code,
- title)
- return u'[[%s:%s:%s]]' % (self.site.family.name,
- self.site.code,
- title)
-
- def __str__(self):
- return self.astext()
-
- def __cmp__(self, other):
- """Test for equality and inequality of Link objects.
-
- Link objects are "equal" if and only if they are on the same site
- and have the same normalized title, including section if any.
-
- Link objects are sortable by site, then namespace, then title.
-
- """
- if not isinstance(other, Link):
- # especially, return -1 if other is None
- return -1
- if not self.site == other.site:
- return cmp(self.site, other.site)
- if self.namespace != other.namespace:
- return cmp(self.namespace, other.namespace)
- return cmp(self.title, other.title)
-
-
-# Utility functions for parsing page titles
-
-def html2unicode(text, ignore = []):
- """Return text, replacing HTML entities by equivalent unicode characters."""
- # This regular expression will match any decimal and hexadecimal entity and
- # also entities that might be named entities.
- entityR = re.compile(
- r'&(#(?P<decimal>\d+)|#x(?P<hex>[0-9a-fA-F]+)|(?P<name>[A-Za-z]+));')
- # These characters are Html-illegal, but sadly you *can* find some of
- # these and converting them to unichr(decimal) is unsuitable
- convertIllegalHtmlEntities = {
- 128 : 8364, # €
- 130 : 8218, # ‚
- 131 : 402, # ƒ
- 132 : 8222, # „
- 133 : 8230, # …
- 134 : 8224, # †
- 135 : 8225, # ‡
- 136 : 710, # ˆ
- 137 : 8240, # ‰
- 138 : 352, # Š
- 139 : 8249, # ‹
- 140 : 338, # Œ
- 142 : 381, # Ž
- 145 : 8216, # ‘
- 146 : 8217, # ’
- 147 : 8220, # “
- 148 : 8221, # ”
- 149 : 8226, # •
- 150 : 8211, # –
- 151 : 8212, # —
- 152 : 732, # ˜
- 153 : 8482, # ™
- 154 : 353, # š
- 155 : 8250, # ›
- 156 : 339, # œ
- 158 : 382, # ž
- 159 : 376 # Ÿ
- }
- #ensuring that illegal   and , which have no known values,
- #don't get converted to unichr(129), unichr(141) or unichr(157)
- ignore = set(ignore) | set([129, 141, 157])
- result = u''
- i = 0
- found = True
- while found:
- text = text[i:]
- match = entityR.search(text)
- if match:
- unicodeCodepoint = None
- if match.group('decimal'):
- unicodeCodepoint = int(match.group('decimal'))
- elif match.group('hex'):
- unicodeCodepoint = int(match.group('hex'), 16)
- elif match.group('name'):
- name = match.group('name')
- if htmlentitydefs.name2codepoint.has_key(name):
- # We found a known HTML entity.
- unicodeCodepoint = htmlentitydefs.name2codepoint[name]
- result += text[:match.start()]
- try:
- unicodeCodepoint=convertIllegalHtmlEntities[unicodeCodepoint]
- except KeyError:
- pass
- if unicodeCodepoint and unicodeCodepoint not in ignore:
- result += unichr(unicodeCodepoint)
- else:
- # Leave the entity unchanged
- result += text[match.start():match.end()]
- i = match.end()
- else:
- result += text
- found = False
- return result
-
-def url2unicode(title, site, site2 = None):
- """Convert url-encoded text to unicode using site's encoding.
-
- If site2 is provided, try its encodings as well. Uses the first encoding
- that doesn't cause an error.
-
- """
- # create a list of all possible encodings for both hint sites
- encList = [site.encoding()] + list(site.encodings())
- if site2 and site2 <> site:
- encList.append(site2.encoding())
- encList += list(site2.encodings())
- firstException = None
- # try to handle all encodings (will probably retry utf-8)
- for enc in encList:
- try:
- t = title.encode(enc)
- t = urllib.unquote(t)
- return unicode(t, enc)
- except UnicodeError, ex:
- if not firstException:
- firstException = ex
- pass
- # Couldn't convert, raise the original exception
- raise firstException
-
+# -*- coding: utf-8 -*-
+"""
+Objects representing various types of MediaWiki pages.
+"""
+#
+# (C) Pywikipedia bot team, 2008
+#
+# Distributed under the terms of the MIT license.
+#
+__version__ = '$Id$'
+
+import pywikibot
+from pywikibot import deprecate_arg
+from pywikibot import config
+import pywikibot.site
+import pywikibot.textlib
+
+import htmlentitydefs
+import logging
+import re
+import sys
+import threading
+import unicodedata
+import urllib
+
+logger = logging.getLogger("wiki")
+
+reNamespace = re.compile("^(.+?) *: *(.*)$")
+
+
+class Page(object):
+ """Page: A MediaWiki page
+
+ This object only implements internally methods that do not require
+ reading from or writing to the wiki. All other methods are delegated
+ to the Site object.
+
+ """
+
+ @deprecate_arg("insite", None)
+ @deprecate_arg("defaultNamespace", None)
+ def __init__(self, source, title=u"", ns=0):
+ """Instantiate a Page object.
+
+ Three calling formats are supported:
+
+ - If the first argument is a Page, create a copy of that object.
+ This can be used to convert an existing Page into a subclass
+ object, such as Category or ImagePage. (If the title is also
+ given as the second argument, creates a copy with that title;
+ this is used when pages are moved.)
+ - If the first argument is a Site, create a Page on that Site
+ using the second argument as the title (may include a section),
+ and the third as the namespace number. The namespace number is
+ mandatory, even if the title includes the namespace prefix. This
+ is the preferred syntax when using an already-normalized title
+ obtained from api.php or a database dump. WARNING: may produce
+ invalid objects if page title isn't in normal form!
+ - If the first argument is a Link, create a Page from that link.
+ This is the preferred syntax when using a title scraped from
+ wikitext, URLs, or another non-normalized source.
+
+ @param source: the source of the page
+ @type source: Link, Page (or subclass), or Site
+ @param title: normalized title of the page; required if source is a
+ Site, ignored otherwise
+ @type title: unicode
+ @param ns: namespace number; required if source is a Site, ignored
+ otherwise
+ @type ns: int
+
+ """
+ if isinstance(source, pywikibot.site.BaseSite):
+ self._site = source
+ if ns not in source.namespaces():
+ raise pywikibot.Error(
+ "Invalid namespace '%i' for site %s."
+ % (ns, source.sitename()))
+ self._ns = ns
+ if ns and not title.startswith(source.namespace(ns)+u":"):
+ title = source.namespace(ns) + u":" + title
+ elif not ns and u":" in title:
+ pos = title.index(u':')
+ nsindex = source.ns_index(title[ :pos])
+ if nsindex:
+ self._ns = nsindex
+ if u"#" in title:
+ title, self._section = title.split(u"#", 1)
+ else:
+ self._section = None
+ if not title:
+ raise pywikibot.Error(
+ "Page object cannot be created from Site without title.")
+ self._title = title
+ elif isinstance(source, Page):
+ # copy all of source's attributes to this object
+ self.__dict__ = source.__dict__
+ if title:
+ # overwrite title
+ if ":" in title:
+ prefix = title[ :title.index(":")]
+ self._ns = site.ns_index(prefix)
+ if self._ns is None:
+ self._ns = 0
+ else:
+ title = title[title.index(":")+1 : ].strip(" _")
+ self._title = "%s:%s" % (
+ self.site().namespace(self._ns),
+ self._title)
+ else:
+ self._ns = 0
+ if "#" in title:
+ self._section = title[title.index("#") + 1 : ].strip(" _")
+ title = title[ : title.index("#")].strip(" _")
+ self._title = title
+ elif isinstance(source, Link):
+ self._site = source.site
+ self._section = source.section
+ self._ns = source.namespace
+ self._title = source.title
+ # reassemble the canonical title from components
+ if self._ns:
+ self._title = "%s:%s" % (self.site().namespace(self._ns),
+ self._title)
+ else:
+ raise pywikibot.Error(
+ "Invalid argument type '%s' in Page constructor: %s"
+ % (type(source), source))
+ if self._section is not None:
+ self._title = self._title + "#" + self._section
+ self._revisions = {}
+
+ def site(self):
+ """Return the Site object for the wiki on which this Page resides."""
+ return self._site
+
+ def namespace(self):
+ """Return the number of the namespace of the page."""
+ return self._ns
+
+ @deprecate_arg("decode", None)
+ @deprecate_arg("savetitle", "asUrl")
+ def title(self, underscore=False, savetitle=False, withNamespace=True,
+ withSection=True, asUrl=False, asLink=False,
+ allowInterwiki=True, forceInterwiki=False, textlink=False,
+ as_filename=False):
+ """Return the title of this Page, as a Unicode string.
+
+ @param underscore: if true, replace all ' ' characters with '_'
+ @param withNamespace: if false, omit the namespace prefix
+ @param withSection: if false, omit the section
+ @param asUrl: if true, quote title as if in an URL
+ @param asLink: if true, return the title in the form of a wikilink
+ @param allowInterwiki: (only used if asLink is true) if true, format
+ the link as an interwiki link if necessary
+ @param forceInterwiki: (only used if asLink is true) if true, always
+ format the link as an interwiki link
+ @param textlink: (only used if asLink is true) if true, place a ':'
+ before Category: and Image: links
+ @param as_filename: if true, replace any characters that are unsafe
+ in filenames
+
+ """
+ title = self._title
+ if not withNamespace and self._ns != 0:
+ title = title.split(u':', 1)[1]
+ if not withSection and self._section:
+ title = title.split(u'#', 1)[0]
+ if underscore or asUrl:
+ title = title.replace(u' ', u'_')
+ if asUrl:
+ encodedTitle = title.encode(self.site().encoding())
+ title = urllib.quote(encodedTitle)
+ if asLink:
+ if forceInterwiki or (allowInterwiki and
+ (self.site().family.name != config.family
+ or self.site().code != config.mylang)):
+ if self.site().family.name != config.family \
+ and self.site().family.name != self.site().code:
+ return u'[[%s:%s:%s]]' % (self.site().family.name,
+ self.site().code,
+ self._title)
+ else:
+ # use this form for sites like commons, where the
+ # code is the same as the family name
+ return u'[[%s:%s]]' % (self.site().code,
+ self._title)
+ elif textlink and (self.isImage() or self.isCategory()):
+ return u'[[:%s]]' % title
+ else:
+ return u'[[%s]]' % title
+ if as_filename:
+ # Replace characters that are not possible in file names on some
+ # systems.
+ # Spaces are possible on most systems, but are bad for URLs.
+ for forbidden in ':*?/\\ ':
+ title = title.replace(forbidden, '_')
+ return title
+
+ @deprecate_arg("decode", None)
+ @deprecate_arg("underscore", None)
+ def section(self):
+ """Return the name of the section this Page refers to.
+
+ The section is the part of the title following a '#' character, if
+ any. If no section is present, return None.
+
+ """
+ if self._section:
+ return self._section
+ else:
+ return None
+
+ def __str__(self):
+ """Return a console representation of the pagelink."""
+ return self.title(asLink=True, forceInterwiki=True
+ ).encode(sys.stderr.encoding)
+
+ def __unicode__(self):
+ return self.title(asLink=True, forceInterwiki=True)
+
+ def __repr__(self):
+ """Return a more complete string representation."""
+ return u"%s(%s)" % (self.__class__.__name__,
+ self.title().encode(sys.stderr.encoding))
+
+ def __cmp__(self, other):
+ """Test for equality and inequality of Page objects.
+
+ Page objects are "equal" if and only if they are on the same site
+ and have the same normalized title, including section if any.
+
+ Page objects are sortable by namespace first, then by title.
+
+ """
+ if not isinstance(other, Page):
+ # especially, return -1 if other is None
+ return -1
+ if not self.site() == other.site():
+ return cmp(self.site(), other.site())
+ if self.namespace() != other.namespace():
+ return cmp(self.namespace(), other.namespace())
+ owntitle = self.title(withNamespace=False)
+ othertitle = other.title(withNamespace=False)
+ return cmp(owntitle, othertitle)
+
+ def __hash__(self):
+ # Pseudo method that makes it possible to store Page objects as keys
+ # in hash-tables. This relies on the fact that the string
+ # representation of an instance can not change after the construction.
+ return hash(unicode(self))
+
+ def autoFormat(self):
+ """Return L{date.autoFormat} dictName and value, if any.
+
+ Value can be a year, date, etc., and dictName is 'YearBC',
+ 'Year_December', or another dictionary name. Please note that two
+ entries may have exactly the same autoFormat, but be in two
+ different namespaces, as some sites have categories with the
+ same names. Regular titles return (None, None).
+
+ """
+ if not hasattr(self, '_autoFormat'):
+ from pywikibot import date
+ self._autoFormat = date.getAutoFormat(
+ self.site().code,
+ self.title(withNamespace=False)
+ )
+ return self._autoFormat
+
+ def isAutoTitle(self):
+ """Return True if title of this Page is in the autoFormat dictionary."""
+ return self.autoFormat()[0] is not None
+
+ @deprecate_arg("throttle", None)
+ @deprecate_arg("nofollow_redirects", None)
+ @deprecate_arg("change_edit_time", None)
+ def get(self, force=False, get_redirect=False, sysop=False):
+ """Return the wiki-text of the page.
+
+ This will retrieve the page from the server if it has not been
+ retrieved yet, or if force is True. This can raise the following
+ exceptions that should be caught by the calling code:
+
+ - NoPage: The page does not exist
+ - IsRedirectPage: The page is a redirect. The argument of the
+ exception is the title of the page it redirects to.
+ - SectionError: The section does not exist on a page with a #
+ link
+
+ @param force: reload all page attributes, including errors.
+ @param get_redirect: return the redirect text, do not follow the
+ redirect, do not raise an exception.
+ @param sysop: if the user has a sysop account, use it to retrieve
+ this page
+
+ """
+ if force:
+ # When forcing, we retry the page no matter what. Old exceptions
+ # do not apply any more.
+ for attr in ['_redirarg', '_getexception']:
+ if hasattr(self, attr):
+ delattr(self,attr)
+ else:
+ # Make sure we re-raise an exception we got on an earlier attempt
+ if hasattr(self, '_redirarg') and not get_redirect:
+ raise pywikibot.IsRedirectPage, self._redirarg
+ elif hasattr(self, '_getexception'):
+ raise self._getexception
+ if force or not hasattr(self, "_revid") \
+ or not self._revid in self._revisions \
+ or self._revisions[self._revid].text is None:
+ self.site().loadrevisions(self, getText=True, sysop=sysop)
+ # TODO: Exception handling for no-page, redirects, etc.
+
+ return self._revisions[self._revid].text
+
+ @deprecate_arg("throttle", None)
+ @deprecate_arg("nofollow_redirects", None)
+ @deprecate_arg("change_edit_time", None)
+ def getOldVersion(self, oldid, force=False, get_redirect=False,
+ sysop=False):
+ """Return text of an old revision of this page; same options as get().
+
+ @param oldid: The revid of the revision desired.
+
+ """
+ if force or not oldid in self._revisions \
+ or self._revisions[oldid].text is None:
+ self.site().loadrevisions(self, getText=True, revids=oldid,
+ sysop=sysop)
+ # TODO: what about redirects, errors?
+ return self._revisions[oldid].text
+
+ def permalink(self):
+ """Return the permalink URL for current revision of this page."""
+ return "%s://%s/%sindex.php?title=%s&oldid=%s" \
+ % (self.site().protocol(),
+ self.site().hostname(),
+ self.site().scriptpath(),
+ self.title(asUrl=True),
+ self.latestRevision())
+
+ def latestRevision(self):
+ """Return the current revision id for this page."""
+ if not hasattr(self, '_revid'):
+ self.site().loadrevisions(self)
+ return self._revid
+
+ def _textgetter(self):
+ """Return the current (edited) wikitext, loading it if necessary."""
+ if not hasattr(self, '_text') or self._text is None:
+ try:
+ self._text = self.get()
+ except pywikibot.NoPage:
+ # TODO: what other exceptions might be returned?
+ self._text = u""
+ return self._text
+
+ def _textsetter(self, value):
+ """Update the edited wikitext"""
+ self._text = unicode(value)
+
+ def _cleartext(self):
+ """Delete the edited wikitext"""
+ if hasattr(self, "_text"):
+ del self._text
+
+ text = property(_textgetter, _textsetter, _cleartext,
+ "The edited wikitext (unicode) of this Page")
+
+ def expand_text(self):
+ """Return the page text with all templates expanded."""
+ req = pywikibot.data.api.Request(action="expandtemplates",
+ text=self.text,
+ title=self.title(withSection=False),
+ site=self.site())
+ result = req.submit()
+ return result["expandtemplates"]["*"]
+
+ def userName(self):
+ """Return name or IP address of last user to edit page."""
+ return self._revisions[self.latestRevision()].user
+
+ def isIpEdit(self):
+ """Return True if last editor was unregistered."""
+ return self._revisions[self.latestRevision()].anon
+
+ def editTime(self):
+ """Return timestamp (in ISO 8601 format) of last revision to page."""
+ return self._revisions[self.latestRevision()].timestamp
+
+ def previousRevision(self):
+ """Return the revision id for the previous revision of this Page."""
+ vh = self.getVersionHistory(revCount=2)
+ revkey = sorted(self._revisions.keys(), reverse=True)[1]
+ return revkey
+
+ def exists(self):
+ """Return True if page exists on the wiki, even if it's a redirect.
+
+ If the title includes a section, return False if this section isn't
+ found.
+
+ """
+ return self.site().page_exists(self)
+
+ def isRedirectPage(self):
+ """Return True if this is a redirect, False if not or not existing."""
+ return self.site().page_isredirect(self)
+
+ def isEmpty(self):
+ """Return True if the page text has less than 4 characters.
+
+ Character count ignores language links and category links.
+ Can raise the same exceptions as get().
+
+ """
+ txt = self.get()
+ txt = pywikibot.textlib.removeLanguageLinks(txt, site = self.site())
+ txt = pywikibot.textlib.removeCategoryLinks(txt, site = self.site())
+ if len(txt) < 4:
+ return True
+ else:
+ return False
+
+ def isTalkPage(self):
+ """Return True if this page is in any talk namespace."""
+ ns = self.namespace()
+ return ns >= 0 and ns % 2 == 1
+
+ def toggleTalkPage(self):
+ """Return other member of the article-talk page pair for this Page.
+
+ If self is a talk page, returns the associated content page;
+ otherwise, returns the associated talk page. The returned page need
+ not actually exist on the wiki.
+
+ Returns None if self is a special page.
+
+ """
+ ns = self.namespace()
+ if ns < 0: # Special page
+ return None
+ if self.isTalkPage():
+ if self.namespace() == 1:
+ return Page(self.site(), self.title(withNamespace=False))
+ else:
+ return Page(self.site(),
+ self.site().namespace(ns - 1) + ':'
+ + self.title(withNamespace=False))
+ else:
+ return Page(self.site(),
+ self.site().namespace(ns + 1) + ':'
+ + self.title(withNamespace=False))
+
+ def isCategory(self):
+ """Return True if the page is a Category, False otherwise."""
+ return self.namespace() == 14
+
+ def isImage(self):
+ """Return True if this is an image description page, False otherwise."""
+ return self.namespace() == 6
+
+ def isDisambig(self):
+ """Return True if this is a disambiguation page, False otherwise.
+
+ Relies on the presence of specific templates, identified in
+ the Family file or on a wiki page, to identify disambiguation
+ pages.
+
+ By default, loads a list of template names from the Family file;
+ if the value in the Family file is None, looks for the list on
+ [[MediaWiki:Disambiguationspage]].
+
+ """
+ if not hasattr(self, "_isDisambig"):
+ if not hasattr(self.site(), "_disambigtemplates"):
+ self.site()._disambigtemplates = \
+ self.site().family.disambig(self.site().code)
+ if self.site()._disambigtemplates is None:
+ try:
+ disambigpages = Page(self.site(),
+ "MediaWiki:Disambiguationspage")
+ self.site()._disambigtemplates = [
+ link.title(withNamespace=False)
+ for link in disambigpages.linkedPages()
+ if link.namespace() == 10
+ ]
+ except NoPage:
+ self.site()._disambigtemplates = ['Disambig']
+ for t in self.templates():
+ if t.title(withNamespace=False) in self.site()._disambigtemplates:
+ self._isDisambig = True
+ break
+ else:
+ self._isDisambig = False
+ return self._isDisambig
+
+ def getReferences(self, follow_redirects=True, withTemplateInclusion=True,
+ onlyTemplateInclusion=False, redirectsOnly=False,
+ namespaces=None):
+ """Return an iterator all pages that refer to or embed the page.
+
+ If you need a full list of referring pages, use
+ C{pages = list(s.getReferences())}
+
+ @param follow_redirects: if True, also iterate pages that link to a
+ redirect pointing to the page.
+ @param withTemplateInclusion: if True, also iterate pages where self
+ is used as a template.
+ @param onlyTemplateInclusion: if True, only iterate pages where self
+ is used as a template.
+ @param redirectsOnly: if True, only iterate redirects to self.
+ @param namespaces: only iterate pages in these namespaces
+
+ """
+ # N.B.: this method intentionally overlaps with backlinks() and
+ # embeddedin(). Depending on the interface, it may be more efficient
+ # to implement those methods in the site interface and then combine
+ # the results for this method, or to implement this method and then
+ # split up the results for the others.
+ return self.site().pagereferences(
+ self, follow_redirects, redirectsOnly,
+ withTemplateInclusion, onlyTemplateInclusion,
+ namespaces)
+
+ def backlinks(self, followRedirects=True, filterRedirects=None,
+ namespaces=None):
+ """Return an iterator for pages that link to this page.
+
+ @param followRedirects: if True, also iterate pages that link to a
+ redirect pointing to the page.
+ @param filterRedirects: if True, only iterate redirects; if False,
+ omit redirects; if None, do not filter
+ @param namespaces: only iterate pages in these namespaces
+
+ """
+ return self.site().pagebacklinks(self, followRedirects, filterRedirects,
+ namespaces)
+
+ def embeddedin(self, filter_redirects=None, namespaces=None):
+ """Return an iterator for pages that embed this page as a template.
+
+ @param filterRedirects: if True, only iterate redirects; if False,
+ omit redirects; if None, do not filter
+ @param namespaces: only iterate pages in these namespaces
+
+ """
+ return self.site().page_embeddedin(self, filter_redirects, namespaces)
+
+ def canBeEdited(self):
+ """Return bool indicating whether this page can be edited.
+
+ This returns True if and only if:
+ - page is unprotected, and bot has an account for this site, or
+ - page is protected, and bot has a sysop account for this site.
+
+ """
+ return self.site().page_can_be_edited(self)
+
+ def botMayEdit(self):
+ """Return True if this page allows bots to edit it.
+
+ This will be True if the page doesn't contain {{bots}} or
+ {{nobots}}, or it contains them and the active bot is allowed to
+ edit this page. (This method is only useful on those sites that
+ recognize the bot-exclusion protocol; on other sites, it will always
+ return True.)
+
+ The framework enforces this restriction by default. It is possible
+ to override this by setting ignore_bot_templates=True in
+ user_config.py, or using page.put(force=True).
+
+ """ # TODO: move this to Site object?
+ if config.ignore_bot_templates: #Check the "master ignore switch"
+ return True
+ try:
+ templates = self.templatesWithParams();
+ except (pywikibot.NoPage,
+ pywikibot.IsRedirectPage,
+ pywikibot.SectionError):
+ return True
+ for template in templates:
+ title = template[0].title(withNamespace=False)
+ if title == 'Nobots':
+ return False
+ elif title == 'Bots':
+ if len(template[1]) == 0:
+ return True
+ else:
+ (ttype, bots) = template[1][0].split('=', 1)
+ bots = bots.split(',')
+ if ttype == 'allow':
+ if 'all' in bots or username in bots:
+ return True
+ else:
+ return False
+ if ttype == 'deny':
+ if 'all' in bots or username in bots:
+ return False
+ else:
+ return True
+ # no restricting template found
+ return True
+
+ def save(self, comment=None, watch=None, minor=True, force=False,
+ async=False, callback=None):
+ """Save the current contents of page's text to the wiki.
+
+ @param comment: The edit summary for the modification (optional, but
+ most wikis strongly encourage its use)
+ @type comment: unicode
+ @param watch: if True, add or if False, remove this Page to/from bot
+ user's watchlist; if None, leave watchlist status unchanged
+ @type watch: bool or None
+ @param minor: if True, mark this edit as minor
+ @type minor: bool
+ @param force: if True, ignore botMayEdit() setting
+ @type force: bool
+ @param async: if True, launch a separate thread to save
+ asynchronously
+ @param callback: a callable object that will be called after the
+ page put operation. This object must take two arguments: (1) a
+ Page object, and (2) an exception instance, which will be None
+ if the page was saved successfully. The callback is intended for
+ use by bots that need to keep track of which saves were
+ successful.
+
+ """
+ if not comment:
+ comment = pywikibot.default_comment # needs to be defined
+ if watch is None:
+ unwatch = False
+ watch = False
+ else:
+ unwatch = not watch
+ if not force and not self.botMayEdit:
+ raise pywikibot.PageNotSaved(
+ "Page %s not saved; editing restricted by {{bots}} template"
+ % self.title(asLink=True))
+ if async:
+ thd = threading.Thread(
+ target=self._save,
+ args=(comment, minor, watch, unwatch, callback)
+ )
+ pywikibot.threadpool.append(thd)
+ thd.start()
+ else:
+ self._save(comment, minor, watch, unwatch, callback)
+
+ def _save(self, comment, minor, watch, unwatch, callback):
+ err = None
+ try:
+ done = self.site().editpage(self, summary=comment, minor=minor,
+ watch=watch, unwatch=unwatch)
+ if not done:
+ logger.warn("Page %s not saved" % self.title(asLink=True))
+ else:
+ logger.info("Page %s saved" % self.title(asLink=True))
+ except pywikibot.Error, err:
+ logger.exception("Error saving page %s" % self.title(asLink=True))
+ if callback:
+ callback(self, err)
+
+ def put(self, newtext, comment=u'', watchArticle=None, minorEdit=True,
+ force=False, async=False, callback=None):
+ """Save the page with the contents of the first argument as the text.
+
+ This method is maintained primarily for backwards-compatibility.
+ For new code, using Page.save() is preferred. See save() method
+ docs for all parameters not listed here.
+
+ @param newtext: The complete text of the revised page.
+ @type newtext: unicode
+
+ """
+ self.text = newtext
+ return self.save(comment, watchArticle, minorEdit, force,
+ async, callback)
+
+ def put_async(self, newtext, comment=u'', watchArticle=None,
+ minorEdit=True, force=False, callback=None):
+ """Put page on queue to be saved to wiki asynchronously.
+
+ Asynchronous version of put (takes the same arguments), which places
+ pages on a queue to be saved by a daemon thread. All arguments are
+ the same as for .put(). This version is maintained solely for
+ backwards-compatibility.
+
+ """
+ return self.put(self, newtext, comment, watchArticle,
+ minorEdit, force, callback, async=True)
+
+ def linkedPages(self):
+ """Iterate Pages that this Page links to.
+
+ Only returns pages from "normal" internal links. Image and category
+ links are omitted unless prefixed with ":". Embedded templates are
+ omitted (but links within them are returned). All interwiki and
+ external links are omitted.
+
+ @return: a generator that yields Page objects.
+
+ """
+ return self.site().pagelinks(self)
+
+ def interwiki(self, expand=True):
+ """Iterate interwiki links in the page text, excluding language links.
+
+ @param expand: if True (default), include interwiki links found in
+ templates transcluded onto this page; if False, only iterate
+ interwiki links found in this page's own wikitext
+ @return: a generator that yields Link objects
+
+ """
+ # This function does not exist in the API, so it has to be
+ # implemented by screen-scraping
+ if expand:
+ text = self.expand_text()
+ else:
+ text = self.text
+ for linkmatch in pywikibot.link_regex.finditer(
+ pywikibot.textlib.removeDisabledParts(text)):
+ linktitle = linkmatch.group("title")
+ link = Link(linktitle, self.site())
+ # only yield links that are to a different site and that
+ # are not language links
+ try:
+ if link.site != self.site():
+ if linktitle.lstrip().startswith(":"):
+ # initial ":" indicates not a language link
+ yield link
+ elif link.site.family != self.site().family:
+ # link to a different family is not a language link
+ yield link
+ except pywikibot.Error:
+ # ignore any links with invalid contents
+ continue
+
+ def langlinks(self):
+ """Iterate all interlanguage links on this page.
+
+ @return: a generator that yields Link objects.
+
+ """
+ return self.site().pagelanglinks(self)
+
+ @deprecate_arg("followRedirects", None)
+ @deprecate_arg("loose", None)
+ def imagelinks(self, followRedirects=None, loose=None):
+ """Iterate ImagePage objects for images displayed on this Page.
+
+ @return: a generator that yields ImagePage objects.
+
+ """
+ return self.site().pageimages(self)
+
+ def templates(self):
+ """Iterate Page objects for templates used on this Page.
+
+ Template parameters are ignored. This method only returns embedded
+ templates, not template pages that happen to be referenced through
+ a normal link.
+
+ """
+ return self.site().pagetemplates(self)
+
+ def templatesWithParams(self):
+ """Iterate templates used on this Page.
+
+ @return: a generator that yields a tuple for each use of a template
+ in the page, with the template Page as the first entry and a list of
+ parameters as the second entry.
+
+ """
+ templates = pywikibot.textlib.extract_templates_and_params(self.text)
+ # backwards-compatibility: convert the dict returned as the second
+ # element into a list in the format used by old scripts
+ result = []
+ for template in templates:
+ args = template[1]
+ positional = []
+ named = {}
+ for key in sorted(args.keys()):
+ try:
+ int(key)
+ except ValueError:
+ named[key] = args[key]
+ else:
+ positional.append(args[key])
+ for name in named:
+ positional.append("%s=%s" % (name, named[name]))
+ result.append((pywikibot.Page(
+ pywikibot.Link(template[0], self.site())),
+ positional))
+ return result
+
+ @deprecate_arg("nofollow_redirects", None)
+ def categories(self, withSortKey=False):
+ """Iterate categories that the article is in.
+
+ @param withSortKey: if True, include the sort key in each Category.
+ @return: a generator that yields Category objects.
+
+ """
+ return self.site().pagecategories(self, withSortKey=withSortKey)
+
+ def extlinks(self):
+ """Iterate all external URLs (not interwiki links) from this page.
+
+ @return: a generator that yields unicode objects containing URLs.
+
+ """
+ return self.site().page_extlinks(self)
+
+ def getRedirectTarget(self):
+ """Return a Page object for the target this Page redirects to.
+
+ If this page is not a redirect page, will raise an IsNotRedirectPage
+ exception. This method also can raise a NoPage exception.
+
+ """
+ if not self.isRedirectPage():
+ raise pywikibot.IsNotRedirectPage
+ if not isinstance(self._redir, Page):
+ self.site().getredirtarget(self)
+ return self._redir
+
+ @deprecate_arg("forceReload", None)
+ def getVersionHistory(self, reverseOrder=False, getAll=False,
+ revCount=500):
+ """Load the version history page and return history information.
+
+ Return value is a list of tuples, where each tuple represents one
+ edit and is built of revision id, edit date/time, user name, and
+ edit summary. Starts with the most current revision, unless
+ reverseOrder is True. Defaults to getting the first revCount edits,
+ unless getAll is True.
+
+ """
+ if getAll:
+ limit = None
+ else:
+ limit = revCount
+ self.site().loadrevisions(self, getText=False, rvdir=reverseOrder,
+ limit=limit)
+ if getAll:
+ revCount = len(self._revisions)
+ return [ ( self._revisions[rev].revid,
+ self._revisions[rev].timestamp,
+ self._revisions[rev].user,
+ self._revisions[rev].comment
+ ) for rev in sorted(self._revisions.keys(),
+ reverse=not reverseOrder)[ : revCount]
+ ]
+
+ def getVersionHistoryTable(self, forceReload=False, reverseOrder=False,
+ getAll=False, revCount=500):
+ """Return the version history as a wiki table."""
+ result = '{| border="1"\n'
+ result += '! oldid || date/time || username || edit summary\n'
+ for oldid, time, username, summary \
+ in self.getVersionHistory(forceReload=forceReload,
+ reverseOrder=reverseOrder,
+ getAll=getAll, revCount=revCount):
+ result += '|----\n'
+ result += '| %s || %s || %s || <nowiki>%s</nowiki>\n'\
+ % (oldid, time, username, summary)
+ result += '|}\n'
+ return result
+
+ def fullVersionHistory(self):
+ """Iterate all previous versions including wikitext.
+
+ @return: A generator that yields tuples consisting of revision ID,
+ edit date/time, user name and content
+ """
+ return self.site().loadrevisions(self, withText=True)
+
+ def contributingUsers(self):
+ """Return a set of usernames (or IPs) of users who edited this page."""
+ edits = self.getVersionHistory()
+ users = set([edit[2] for edit in edits])
+ return users
+
+ @deprecate_arg("throttle", None)
+ def move(self, newtitle, reason=None, movetalkpage=True, sysop=False,
+ deleteAndMove=False, safe=True):
+ """Move this page to a new title.
+
+ @param newtitle: The new page title.
+ @param reason: The edit summary for the move.
+ @param movetalkpage: If true, move this page's talk page (if it exists)
+ @param sysop: Try to move using sysop account, if available
+ @param deleteAndMove: if move succeeds, delete the old page
+ (usually requires sysop privileges, depending on wiki settings)
+ @param safe: If false, attempt to delete existing page at newtitle
+ (if there is one) and then move this page to that title
+
+ """
+ if reason is None:
+ logger.info(u'Moving %s to [[%s]].'
+ % (self.title(asLink=True), newtitle))
+ reason = pywikibot.input(u'Please enter a reason for the move:')
+ # TODO: implement "safe" parameter
+ # TODO: implement "sysop" parameter
+ return self.site().movepage(self, newtitle, reason,
+ movetalk=movetalkpage,
+ noredirect=deleteAndMove)
+
+ @deprecate_arg("throttle", None)
+ def delete(self, reason=None, prompt=True, throttle=None, mark=False):
+ """Deletes the page from the wiki. Requires administrator status.
+
+ @param reason: The edit summary for the deletion.
+ @param prompt: If true, prompt user for confirmation before deleting.
+ @param mark: if true, and user does not have sysop rights, place a
+ speedy-deletion request on the page instead.
+
+ """
+ if reason is None:
+ logger.info(u'Deleting %s.' % (self.title(asLink=True)))
+ reason = pywikibot.input(u'Please enter a reason for the deletion:')
+ answer = u'y'
+ if prompt and not hasattr(self.site(), '_noDeletePrompt'):
+ answer = pywikibot.inputChoice(u'Do you want to delete %s?'
+ % self.title(asLink = True, forceInterwiki = True),
+ ['Yes', 'No', 'All'],
+ ['Y', 'N', 'A'],
+ 'N')
+ if answer in ['a', 'A']:
+ answer = 'y'
+ self.site()._noDeletePrompt = True
+ if answer in ['y', 'Y']:
+ return self.site().delete(self, reason, mark=mark)
+
+ def loadDeletedRevisions(self):
+ """Retrieve all deleted revisions for this Page from Special/Undelete.
+
+ Stores all revisions' timestamps, dates, editors and comments in
+ self._deletedRevs attribute.
+
+ @return: list of timestamps (which can be used to retrieve revisions
+ later on).
+
+ """
+ return self.site().loadDeletedRevisions(self)
+
+ def getDeletedRevision(self, timestamp, retrieveText=False):
+ """Return a particular deleted revision by timestamp.
+
+ @return: a list of [date, editor, comment, text, restoration
+ marker]. text will be None, unless retrieveText is True (or has
+ been retrieved earlier). If timestamp is not found, returns
+ None.
+
+ """
+ return self.site().getDeletedRevision(self, timestamp,
+ getText=retrieveText)
+
+ def markDeletedRevision(self, timestamp, undelete=True):
+ """Mark the revision identified by timestamp for undeletion.
+
+ @param undelete: if False, mark the revision to remain deleted.
+
+ """
+ if self._deletedRevs == None:
+ self.loadDeletedRevisions()
+ if not self._deletedRevs.has_key(timestamp):
+ #TODO: Throw an exception?
+ return None
+ self._deletedRevs[timestamp][4] = undelete
+ self._deletedRevsModified = True
+
+ @deprecate_arg("throttle", None)
+ def undelete(self, comment=None):
+ """Undelete revisions based on the markers set by previous calls.
+
+ If no calls have been made since loadDeletedRevisions(), everything
+ will be restored.
+
+ Simplest case::
+ Page(...).undelete('This will restore all revisions')
+
+ More complex::
+ pg = Page(...)
+ revs = pg.loadDeletedRevsions()
+ for rev in revs:
+ if ... #decide whether to undelete a revision
+ pg.markDeletedRevision(rev) #mark for undeletion
+ pg.undelete('This will restore only selected revisions.')
+
+ @param comment: The undeletion edit summary.
+
+ """
+ if comment is None:
+ logger.info(u'Preparing to undelete %s.'
+ % (self.title(asLink=True)))
+ comment = pywikibot.input(
+ u'Please enter a reason for the undeletion:')
+ return self.site().undelete(self, comment)
+
+ @deprecate_arg("throttle", None)
+ def protect(self, edit='sysop', move='sysop', unprotect=False,
+ reason=None, prompt=True):
+ """(Un)protect a wiki page. Requires administrator status.
+
+ Valid protection levels (in MediaWiki 1.12) are '' (equivalent to
+ 'none'), 'autoconfirmed', and 'sysop'.
+
+ @param edit: Level of edit protection
+ @param move: Level of move protection
+ @param unprotect: If true, unprotect the page (equivalent to setting
+ all protection levels to '')
+ @param reason: Edit summary.
+ @param prompt: If true, ask user for confirmation.
+
+ """
+ if reason is None:
+ if unprotect:
+ un = u'un'
+ else:
+ un = u''
+ logger.info(u'Preparing to %sprotect %s.'
+ % (un, self.title(asLink=True)))
+ reason = pywikibot.input(u'Please enter a reason for the action:')
+ if unprotect:
+ edit = move = ""
+ answer = 'y'
+ if prompt and not hasattr(self.site(), '_noProtectPrompt'):
+ answer = pywikibot.inputChoice(
+ u'Do you want to change the protection level of %s?'
+ % self.title(asLink=True, forceInterwiki = True),
+ ['Yes', 'No', 'All'], ['Y', 'N', 'A'], 'N')
+ if answer in ['a', 'A']:
+ answer = 'y'
+ self.site()._noProtectPrompt = True
+ if answer in ['y', 'Y']:
+ return self.site().protect(self, edit, move, reason)
+
+ def change_category(article, oldCat, newCat, comment=None, sortKey=None,
+ inPlace=True):
+ """Remove page from oldCat and add it to newCat.
+
+ oldCat and newCat should be Category objects.
+ If newCat is None, the category will be removed.
+
+ """ # TODO: document remaining arguments
+ cats = self.categories(get_redirect=True)
+ site = self.site()
+ changesMade = False
+
+ if not self.canBeEdited():
+ pywikibot.output(u"Can't edit %s, skipping it..."
+ % self.title(asLink=True))
+ return False
+ if inPlace == True:
+ newtext = pywikibot.textlib.replaceCategoryInPlace(
+ self.text, oldCat, newCat)
+ if newtext == self.text:
+ pywikibot.output(
+ u'No changes in made in page %s.'
+ % self.title(asLink=True))
+ return False
+ try:
+ self.put(newtext, comment)
+ return True
+ except pywikibot.EditConflict:
+ pywikibot.output(
+ u'Skipping %s because of edit conflict'
+ % self.title(asLink=True))
+ except pywikibot.LockedPage:
+ pywikibot.output(u'Skipping locked page %s'
+ % self.title(asLink=True))
+ except pywikibot.SpamfilterError, error:
+ pywikibot.output(
+ u'Changing page %s blocked by spam filter (URL=%s)'
+ % (self.title(asLink=True), error.url))
+ except pywikibot.NoUsername:
+ pywikibot.output(
+ u"Page %s not saved; sysop privileges required."
+ % self.title(asLink=True))
+ except pywikibot.PageNotSaved, error:
+ pywikibot.output(u"Saving page %s failed: %s"
+ % (self.title(asLink=True), error.message))
+ return False
+
+ # This loop will replace all occurrences of the category to be changed,
+ # and remove duplicates.
+ newCatList = []
+ newCatSet = set()
+ for i in range(len(cats)):
+ cat = cats[i]
+ if cat == oldCat:
+ changesMade = True
+ if not sortKey:
+ sortKey = cat.sortKey
+ if newCat:
+ if newCat.title() not in newCatSet:
+ newCategory = Category(site, newCat.title(),
+ sortKey=sortKey)
+ newCatSet.add(newCat.title())
+ newCatList.append(newCategory)
+ elif cat.title() not in newCatSet:
+ newCatSet.add(cat.title())
+ newCatList.append(cat)
+
+ if not changesMade:
+ pywikibot.output(u'ERROR: %s is not in category %s!'
+ % (self.title(asLink=True), oldCat.title()))
+ else:
+ try:
+ text = pywikibot.textlib.replaceCategoryLinks(self.text,
+ newCatList)
+ except ValueError:
+ # Make sure that the only way replaceCategoryLinks() can return
+ # a ValueError is in the case of interwiki links to self.
+ pywikibot.output(
+ u'Skipping %s because of interwiki link to self' % self)
+ try:
+ self.put(text, comment)
+ except pywikibot.EditConflict:
+ pywikibot.output(
+ u'Skipping %s because of edit conflict' % self.title())
+ except pywikibot.SpamfilterError, e:
+ pywikibot.output(
+ u'Skipping %s because of blacklist entry %s'
+ % (self.title(), e.url))
+ except pywikibot.LockedPage:
+ pywikibot.output(
+ u'Skipping %s because page is locked' % self.title())
+ except pywikibot.PageNotSaved, error:
+ pywikibot.output(u"Saving page %s failed: %s"
+ % (self.title(asLink=True), error.message))
+
+######## DEPRECATED METHODS ########
+
+ def encoding(self):
+ """DEPRECATED: use Site.encoding() instead"""
+ logger.debug(u"Page.encoding() is deprecated; use Site.encoding().")
+ return self.site().encoding()
+
+ def titleWithoutNamespace(self, underscore=False):
+ """DEPRECATED: use self.title(withNamespace=False) instead."""
+ logger.debug(
+ u"Page.titleWithoutNamespace() method is deprecated.")
+ return self.title(underscore=underscore, withNamespace=False,
+ withSection=False)
+
+ def titleForFilename(self):
+ """DEPRECATED: use self.title(as_filename=True) instead."""
+ logger.debug(
+ u"Page.titleForFilename() method is deprecated.")
+ return self.title(as_filename=True)
+
+ def sectionFreeTitle(self, underscore=False):
+ """DEPRECATED: use self.title(withSection=False) instead."""
+ logger.debug(
+ u"Page.sectionFreeTitle() method is deprecated.")
+ return self.title(underscore=underscore, withSection=False)
+
+ def aslink(self, forceInterwiki=False, textlink=False, noInterwiki=False):
+ """DEPRECATED: use self.title(asLink=True) instead."""
+ logger.debug(u"Page.aslink() method is deprecated.")
+ return self.title(asLink=True, forceInterwiki=forceInterwiki,
+ allowInterwiki=not noInterwiki, textlink=textlink)
+
+ def urlname(self):
+ """Return the Page title encoded for use in an URL.
+
+ DEPRECATED: use self.title(asUrl=True) instead.
+
+ """
+ logger.debug(u"Page.urlname() method is deprecated.")
+ return self.title(asUrl=True)
+
+####### DISABLED METHODS (warnings provided) ######
+ # these methods are easily replaced by editing the page's text using
+ # textlib methods and then using put() on the result.
+
+ def removeImage(self, image, put=False, summary=None, safe=True):
+ """Old method to remove all instances of an image from page."""
+ logger.warning(u"Page.removeImage() is no longer supported.")
+
+ def replaceImage(self, image, replacement=None, put=False, summary=None,
+ safe=True):
+ """Old method to replace all instances of an image with another."""
+ logger.warning(u"Page.replaceImage() is no longer supported.")
+
+
+class ImagePage(Page):
+ """A subclass of Page representing an image descriptor wiki page.
+
+ Supports the same interface as Page, with the following added methods:
+
+ getImagePageHtml : Download image page and return raw HTML text.
+ fileURL : Return the URL for the image described on this
+ page.
+ fileIsOnCommons : Return True if image stored on Wikimedia
+ Commons.
+ fileIsShared : Return True if image stored on Wikitravel
+ shared repository.
+ getFileMd5Sum : Return image file's MD5 checksum.
+ getFileVersionHistory : Return the image file's version history.
+ getFileVersionHistoryTable: Return the version history in the form of a
+ wiki table.
+ usingPages : Iterate Pages on which the image is displayed.
+
+ """
+ def __init__(self, source, title=u"", insite=None):
+ Page.__init__(self, source, title, 6)
+ if self.namespace() != 6:
+ raise ValueError(u"'%s' is not in the image namespace!" % title)
+
+ def getImagePageHtml(self):
+ """
+ Download the image page, and return the HTML, as a unicode string.
+
+ Caches the HTML code, so that if you run this method twice on the
+ same ImagePage object, the page will only be downloaded once.
+ """
+ if not hasattr(self, '_imagePageHtml'):
+ from pywikibot.data import http
+ path = "%s/index.php?title=%s" \
+ % (self.site().scriptpath(), self.title(asUrl=True))
+ self._imagePageHtml = http.request(self.site(), path)
+ return self._imagePageHtml
+
+ def fileUrl(self):
+ """Return the URL for the image described on this page."""
+ # TODO add scaling option?
+ if not hasattr(self, '_imageinfo'):
+ self._imageinfo = self.site().getimageinfo(self) #FIXME
+ return self._imageinfo['url']
+
+ def fileIsOnCommons(self):
+ """Return True if the image is stored on Wikimedia Commons"""
+ return self.fileUrl().startswith(
+ 'http://upload.wikimedia.org/wikipedia/commons/')
+
+ def fileIsShared(self):
+ """Return True if image is stored on any known shared repository."""
+ # as of now, the only known repositories are commons and wikitravel
+ if 'wikitravel_shared' in self.site().shared_image_repository():
+ return self.fileUrl().startswith(
+ u'http://wikitravel.org/upload/shared/')
+ return self.fileIsOnCommons()
+
+ def getFileMd5Sum(self):
+ """Return image file's MD5 checksum."""
+ logger.debug(
+ "ImagePage.getFileMd5Sum() is deprecated; use getFileSHA1Sum().")
+# FIXME: MD5 might be performed on incomplete file due to server disconnection
+# (see bug #1795683).
+ import md5, urllib
+ f = urllib.urlopen(self.fileUrl())
+ # TODO: check whether this needs a User-Agent header added
+ md5Checksum = md5.new(f.read()).hexdigest()
+ f.close()
+ return md5Checksum
+
+ def getFileSHA1Sum(self):
+ """Return image file's SHA1 checksum."""
+ if not hasattr(self, '_imageinfo'):
+ self._imageinfo = self.site().getimageinfo(self) #FIXME
+ return self._imageinfo['sha1']
+
+ def getFileVersionHistory(self):
+ """Return the image file's version history.
+
+ @return: An iterator yielding tuples containing (timestamp,
+ username, resolution, filesize, comment).
+
+ """
+ #TODO; return value may need to change
+ return self.site().getimageinfo(self, history=True) #FIXME
+
+ def getFileVersionHistoryTable(self):
+ """Return the version history in the form of a wiki table."""
+ lines = []
+ #TODO: if getFileVersionHistory changes, make sure this follows it
+ for (datetime, username, resolution, size, comment) \
+ in self.getFileVersionHistory():
+ lines.append('| %s || %s || %s || %s || <nowiki>%s</nowiki>' \
+ % (datetime, username, resolution, size, comment))
+ return u'{| border="1"\n! date/time || username || resolution || size || edit summary\n|----\n' + u'\n|----\n'.join(lines) + '\n|}'
+
+ def usingPages(self):
+ """Yield Pages on which the image is displayed."""
+ return self.site().getimageusage(self)
+
+
+class Category(Page):
+ """A page in the Category: namespace"""
+
+ @deprecate_arg("sortKey", None)
+ def __init__(self, source, title=u"", insite=None):
+ """All parameters are the same as for Page() constructor.
+
+ """
+ Page.__init__(self, source, title, 14)
+ if self.namespace() != 14:
+ raise ValueError(u"'%s' is not in the category namespace!"
+ % title)
+
+ @deprecate_arg("forceInterwiki", None)
+ @deprecate_arg("textlink", None)
+ @deprecate_arg("noInterwiki", None)
+ def aslink(self, sortKey=u''):
+ """Return a link to place a page in this Category.
+
+ Use this only to generate a "true" category link, not for interwikis
+ or text links to category pages.
+
+ @param sortKey: The sort key for the article to be placed in this
+ Category; if omitted, default sort key is used.
+ @type sortKey: (optional) unicode
+
+ """
+ if sortKey:
+ titleWithSortKey = '%s|%s' % (self.title(withSection=False),
+ self.sortKey)
+ else:
+ titleWithSortKey = self.title(withSection=False)
+ return '[[%s]]' % titleWithSortKey
+
+ @deprecate_arg("startFrom", None)
+ @deprecate_arg("cacheResults", None)
+ def subcategories(self, recurse=False):
+ """Iterate all subcategories of the current category.
+
+ @param recurse: if not False or 0, also iterate subcategories of
+ subcategories. If an int, limit recursion to this number of
+ levels. (Example: recurse=1 will iterate direct subcats and
+ first-level sub-sub-cats, but no deeper.)
+ @type recurse: int or bool
+
+ """
+ if not isinstance(recurse, bool) and recurse:
+ recurse = recurse - 1
+ if not hasattr(self, "_subcats"):
+ self._subcats = []
+ for member in self.site().categorymembers(self, namespaces=[14]):
+ subcat = Category(self.site(), member.title())
+ self._subcats.append(subcat)
+ yield subcat
+ if recurse:
+ for item in subcat.subcategories(recurse):
+ yield item
+ else:
+ for subcat in self._subcats:
+ yield subcat
+ if recurse:
+ for item in subcat.subcategories(recurse):
+ yield item
+
+ @deprecate_arg("startFrom", None)
+ def articles(self, recurse=False):
+ """
+ Yields all articles in the current category.
+
+ @param recurse: if not False or 0, also iterate articles in
+ subcategories. If an int, limit recursion to this number of
+ levels. (Example: recurse=1 will iterate articles in first-level
+ subcats, but no deeper.)
+ @type recurse: int or bool
+
+ """
+ namespaces = [x for x in self.site().namespaces().keys()
+ if x>=0 and x!=14]
+ for member in self.site().categorymembers(self,
+ namespaces=namespaces):
+ yield member
+ if recurse:
+ if not isinstance(recurse, bool) and recurse:
+ recurse = recurse - 1
+ for subcat in self.subcategories():
+ for article in subcat.articles(recurse):
+ yield article
+
+ def isEmptyCategory(self):
+ """Return True if category has no members (including subcategories)."""
+ for member in self.site().categorymembers(self, limit=1):
+ return False
+ return True
+
+ def copyTo(self, catname):
+ """
+ Copy text of category page to a new page. Does not move contents.
+
+ @param catname: New category title (without namespace)
+ @return: True if copying was successful, False if target page
+ already existed.
+
+ """
+ # This seems far too specialized to be in the top-level framework
+ catname = self.site().category_namespace() + ':' + catname
+ targetCat = Category(self.site(), catname)
+ if targetCat.exists():
+ logger.warn('Target page %s already exists!'
+ % targetCat.title())
+ return False
+ else:
+ logger.info('Moving text from %s to %s.'
+ % (self.title(), targetCat.title()))
+ authors = ', '.join(self.contributingUsers())
+ creationSummary = pywikibot.translate(
+ self.site(), msg_created_for_renaming
+ ) % (self.title(), authors)
+ targetCat.put(self.get(), creationSummary)
+ return True
+
+ def copyAndKeep(self, catname, cfdTemplates):
+ """Copy partial category page text (not contents) to a new title.
+
+ Like copyTo above, except this removes a list of templates (like
+ deletion templates) that appear in the old category text. It also
+ removes all text between the two HTML comments BEGIN CFD TEMPLATE
+ and END CFD TEMPLATE. (This is to deal with CFD templates that are
+ substituted.)
+
+ Returns true if copying was successful, false if target page already
+ existed.
+
+ @param catname: New category title (without namespace)
+ @param cfdTemplates: A list (or iterator) of templates to be removed
+ from the page text
+ @return: True if copying was successful, False if target page
+ already existed.
+
+ """
+ # I don't see why we need this as part of the framework either
+ catname = self.site().category_namespace() + ':' + catname
+ targetCat = Category(self.site(), catname)
+ if targetCat.exists():
+ logger.warn('Target page %s already exists!'
+ % targetCat.title())
+ return False
+ else:
+ logger.info('Moving text from %s to %s.'
+ % (self.title(), targetCat.title()))
+ authors = ', '.join(self.contributingUsers())
+ creationSummary = pywikibot.translate(
+ self.site(), msg_created_for_renaming
+ ) % (self.title(), authors)
+ newtext = self.get()
+ for regexName in cfdTemplates:
+ matchcfd = re.compile(r"{{%s.*?}}" % regexName, re.IGNORECASE)
+ newtext = matchcfd.sub('',newtext)
+ matchcomment = re.compile(
+ r"<!--BEGIN CFD TEMPLATE-->.*?<!--END CFD TEMPLATE-->",
+ re.IGNORECASE | re.MULTILINE | re.DOTALL)
+ newtext = matchcomment.sub('', newtext)
+ pos = 0
+ while (newtext[pos:pos+1] == "\n"):
+ pos = pos + 1
+ newtext = newtext[pos:]
+ targetCat.put(newtext, creationSummary)
+ return True
+
+#### DEPRECATED METHODS ####
+ def subcategoriesList(self, recurse=False):
+ """DEPRECATED: Equivalent to list(self.subcategories(...))"""
+ logger.debug("Category.subcategoriesList() method is deprecated.")
+ return sorted(list(set(self.subcategories(recurse))))
+
+ def articlesList(self, recurse=False):
+ """DEPRECATED: equivalent to list(self.articles(...))"""
+ logger.debug("Category.articlesList() method is deprecated.")
+ return sorted(list(set(self.articles(recurse))))
+
+ def supercategories(self):
+ """DEPRECATED: equivalent to self.categories()"""
+ logger.debug("Category.supercategories() method is deprecated.")
+ return self.categories()
+
+ def supercategoriesList(self):
+ """DEPRECATED: equivalent to list(self.categories(...))"""
+ logger.debug("Category.articlesList() method is deprecated.")
+ return sorted(list(set(self.categories())))
+
+
+class Revision(object):
+ """A structure holding information about a single revision of a Page."""
+ def __init__(self, revid, timestamp, user, anon=False, comment=u"",
+ text=None, minor=False):
+ """All parameters correspond to object attributes (e.g., revid
+ parameter is stored as self.revid)
+
+ @param revid: Revision id number
+ @type revid: int
+ @param text: Revision wikitext.
+ @type text: unicode, or None if text not yet retrieved
+ @param timestamp: Revision time stamp (in ISO 8601 format)
+ @type timestamp: unicode
+ @param user: user who edited this revision
+ @type user: unicode
+ @param anon: user is unregistered
+ @type anon: bool
+ @param comment: edit comment text
+ @type comment: unicode
+ @param minor: edit flagged as minor
+ @type minor: bool
+
+ """
+ self.revid = revid
+ self.text = text
+ self.timestamp = timestamp
+ self.user = user
+ self.anon = anon
+ self.comment = comment
+ self.minor = minor
+
+
+class Link(object):
+ """A Mediawiki link (local or interwiki)
+
+ Has the following attributes:
+
+ - site: The Site object for the wiki linked to
+ - namespace: The namespace of the page linked to (int)
+ - title: The title of the page linked to (unicode); does not include
+ namespace or section
+ - section: The section of the page linked to (unicode or None); this
+ contains any text following a '#' character in the title
+ - anchor: The anchor text (unicode or None); this contains any text
+ following a '|' character inside the link
+
+ """
+ illegal_titles_pattern = re.compile(
+ # Matching titles will be held as illegal.
+ u'''[^ %!\"$&'()*,\\-.\\/0-9:;=?@A-Z\\\\^_`a-z~\u0080-\uFFFF+]'''
+ # URL percent encoding sequences interfere with the ability
+ # to round-trip titles -- you can't link to them consistently.
+ u'|%[0-9A-Fa-f]{2}'
+ # XML/HTML character references produce similar issues.
+ u'|&[A-Za-z0-9\x80-\xff]+;'
+ u'|&#[0-9]+;'
+ u'|&#x[0-9A-Fa-f]+;'
+ )
+
+ def __init__(self, text, source=None, defaultNamespace=0):
+ """Constructor
+
+ @param text: the link text (everything appearing between [[ and ]]
+ on a wiki page)
+ @type text: unicode
+ @param source: the Site on which the link was found (not necessarily
+ the site to which the link refers)
+ @type source: Site
+ @param defaultNamespace: a namespace to use if the link does not
+ contain one (defaults to 0)
+ @type defaultNamespace: int
+
+ """
+ self._text = text
+ self._source = source
+ self._defaultns = defaultNamespace
+
+ def parse(self):
+ """Parse text; called internally when accessing attributes"""
+
+ # First remove the anchor, which is stored unchanged, if there is one
+ if u"|" in self._text:
+ self._text, self._anchor = self._text.split(u"|", 1)
+ else:
+ self._anchor = None
+
+ if self._source is None:
+ self._source = pywikibot.Site()
+ self._site = self._source
+
+ # Clean up the name, it can come from anywhere.
+ # Convert HTML entities to unicode
+ t = html2unicode(self._text)
+
+ # Convert URL-encoded characters to unicode
+ t = url2unicode(t, site=self._site)
+
+ # Normalize unicode string to a NFC (composed) format to allow proper
+ # string comparisons. According to
+ # http://svn.wikimedia.org/viewvc/mediawiki/branches/REL1_6/phase3/includes/n…
+ # the mediawiki code normalizes everything to NFC, not NFKC (which
+ # might result in information loss).
+ t = unicodedata.normalize('NFC', t)
+
+ # This code was adapted from Title.php : secureAndSplit()
+ #
+ if u'\ufffd' in t:
+ raise pywikibot.Error("Title contains illegal char (\\uFFFD)")
+ self._namespace = self._defaultns
+
+ # Replace underscores by spaces
+ t = t.replace(u"_", u" ")
+ # replace multiple spaces and underscores with a single space
+ while u" " in t: t = t.replace(u" ", u" ")
+ # Strip spaces at both ends
+ t = t.strip(" ")
+ # Remove left-to-right and right-to-left markers.
+ t = t.replace(u"\u200e", u"").replace(u"\u200f", u"")
+
+ firstPass = True
+ while u":" in t:
+ # Initial colon indicates main namespace rather than default
+ if t.startswith(u":"):
+ self._namespace = 0
+ # remove the colon but continue processing
+ # remove any subsequent whitespace
+ t = t.lstrip(u":").lstrip(u" ")
+ continue
+
+ fam = self._site.family
+ prefix = t[ :t.index(u":")].lower()
+ ns = self._site.ns_index(prefix)
+ if ns:
+ # Ordinary namespace
+ t = t[t.index(u":"): ].lstrip(u":").lstrip(u" ")
+ self._namespace = ns
+ break
+ if prefix in fam.langs.keys()\
+ or prefix in fam.get_known_families(site=self._site):
+ # looks like an interwiki link
+ if not firstPass:
+ # Can't make a local interwiki link to an interwiki link.
+ raise pywikibot.Error(
+ "Improperly formatted interwiki link '%s'"
+ % self._text)
+ t = t[t.index(u":"): ].lstrip(u":").lstrip(u" ")
+ if prefix in fam.langs.keys():
+ newsite = pywikibot.Site(prefix, fam)
+ else:
+ otherlang = self._site.code
+ familyName = fam.get_known_families(site=self._site)[prefix]
+ if familyName in ['commons', 'meta']:
+ otherlang = familyName
+ try:
+ newsite = pywikibot.Site(otherlang, familyName)
+ except ValueError:
+ raise pywikibot.Error("""\
+%s is not a local page on %s, and the %s family is
+not supported by PyWikiBot!"""
+ % (title, self._site(), familyName))
+
+ # Redundant interwiki prefix to the local wiki
+ if newsite == self._site:
+ if not t:
+ # Can't have an empty self-link
+ raise pywikibot.Error(
+ "Invalid link title: '%s'" % self._text)
+ firstPass = False
+ continue
+ self._site = newsite
+ else:
+ break # text before : doesn't match any known prefix
+
+ if u"#" in t:
+ t, sec = t.split(u'#', 1)
+ t, self._section = t.rstrip(), sec.lstrip()
+ else:
+ self._section = None
+
+ # Reject illegal characters.
+ m = Link.illegal_titles_pattern.search(t)
+ if m:
+ raise pywikibot.Error(
+ u"Invalid title: contains illegal char(s) '%s'" % m.group(0))
+
+ # Pages with "/./" or "/../" appearing in the URLs will
+ # often be unreachable due to the way web browsers deal
+ #* with 'relative' URLs. Forbid them explicitly.
+
+ if u'.' in t and (
+ t == u'.' or t == u'..'
+ or t.startswith(u"./")
+ or t.startswith(u"../")
+ or u"/./" in t
+ or u"/../" in t
+ or t.endswith(u"/.")
+ or t.endswith(u"/..")
+ ):
+ raise pywikibot.Error(
+ "Invalid title (contains . / combinations): '%s'"
+ % self._text)
+
+ # Magic tilde sequences? Nu-uh!
+ if u"~~~" in t:
+ raise pywikibot.Error("Invalid title (contains ~~~): '%s'" % self._text)
+
+ if self._namespace != -1 and len(t) > 255:
+ raise pywikibot.Error("Invalid title (over 255 bytes): '%s'" % t)
+
+ if self._site.case() == 'first-letter':
+ t = t[:1].upper() + t[1:]
+
+ # Can't make a link to a namespace alone...
+ # "empty" local links can only be self-links
+ # with a fragment identifier.
+ if not t and self._site == self._source and self._namespace != 0:
+ raise ValueError("Invalid link (no page title): '%s'" % self._text)
+
+ self._title = t
+
+ # define attributes, to be evaluated lazily
+
+ @property
+ def site(self):
+ if not hasattr(self, "_site"):
+ self.parse()
+ return self._site
+
+ @property
+ def namespace(self):
+ if not hasattr(self, "_namespace"):
+ self.parse()
+ return self._namespace
+
+ @property
+ def title(self):
+ if not hasattr(self, "_title"):
+ self.parse()
+ return self._title
+
+ @property
+ def section(self):
+ if not hasattr(self, "_section"):
+ self.parse()
+ return self._section
+
+ @property
+ def anchor(self):
+ if not hasattr(self, "_anchor"):
+ self.parse()
+ return self._anchor
+
+ def astext(self, onsite=None):
+ """Return a text representation of the link.
+
+ @param onsite: if specified, present as a (possibly interwiki) link
+ from the given site; otherwise, present as an internal link on
+ the source site.
+
+ """
+ if onsite is None:
+ onsite = self.site
+ title = self.title
+ if self.namespace:
+ title = onsite.namespace(self.namespace) + ":" + title
+ if self.section:
+ title = title + "#" + self.section
+ if onsite == self.site:
+ return u'[[%s]]' % title
+ if onsite.family == self.site.family:
+ return u'[[%s:%s]]' % (self.site.code, title)
+ if self.site.family.name == self.site.code:
+ # use this form for sites like commons, where the
+ # code is the same as the family name
+ return u'[[%s:%s]]' % (self.site.code,
+ title)
+ return u'[[%s:%s:%s]]' % (self.site.family.name,
+ self.site.code,
+ title)
+
+ def __str__(self):
+ return self.astext()
+
+ def __cmp__(self, other):
+ """Test for equality and inequality of Link objects.
+
+ Link objects are "equal" if and only if they are on the same site
+ and have the same normalized title, including section if any.
+
+ Link objects are sortable by site, then namespace, then title.
+
+ """
+ if not isinstance(other, Link):
+ # especially, return -1 if other is None
+ return -1
+ if not self.site == other.site:
+ return cmp(self.site, other.site)
+ if self.namespace != other.namespace:
+ return cmp(self.namespace, other.namespace)
+ return cmp(self.title, other.title)
+
+
+# Utility functions for parsing page titles
+
+def html2unicode(text, ignore = []):
+ """Return text, replacing HTML entities by equivalent unicode characters."""
+ # This regular expression will match any decimal and hexadecimal entity and
+ # also entities that might be named entities.
+ entityR = re.compile(
+ r'&(#(?P<decimal>\d+)|#x(?P<hex>[0-9a-fA-F]+)|(?P<name>[A-Za-z]+));')
+ # These characters are Html-illegal, but sadly you *can* find some of
+ # these and converting them to unichr(decimal) is unsuitable
+ convertIllegalHtmlEntities = {
+ 128 : 8364, # €
+ 130 : 8218, # ‚
+ 131 : 402, # ƒ
+ 132 : 8222, # „
+ 133 : 8230, # …
+ 134 : 8224, # †
+ 135 : 8225, # ‡
+ 136 : 710, # ˆ
+ 137 : 8240, # ‰
+ 138 : 352, # Š
+ 139 : 8249, # ‹
+ 140 : 338, # Œ
+ 142 : 381, # Ž
+ 145 : 8216, # ‘
+ 146 : 8217, # ’
+ 147 : 8220, # “
+ 148 : 8221, # ”
+ 149 : 8226, # •
+ 150 : 8211, # –
+ 151 : 8212, # —
+ 152 : 732, # ˜
+ 153 : 8482, # ™
+ 154 : 353, # š
+ 155 : 8250, # ›
+ 156 : 339, # œ
+ 158 : 382, # ž
+ 159 : 376 # Ÿ
+ }
+ #ensuring that illegal   and , which have no known values,
+ #don't get converted to unichr(129), unichr(141) or unichr(157)
+ ignore = set(ignore) | set([129, 141, 157])
+ result = u''
+ i = 0
+ found = True
+ while found:
+ text = text[i:]
+ match = entityR.search(text)
+ if match:
+ unicodeCodepoint = None
+ if match.group('decimal'):
+ unicodeCodepoint = int(match.group('decimal'))
+ elif match.group('hex'):
+ unicodeCodepoint = int(match.group('hex'), 16)
+ elif match.group('name'):
+ name = match.group('name')
+ if htmlentitydefs.name2codepoint.has_key(name):
+ # We found a known HTML entity.
+ unicodeCodepoint = htmlentitydefs.name2codepoint[name]
+ result += text[:match.start()]
+ try:
+ unicodeCodepoint=convertIllegalHtmlEntities[unicodeCodepoint]
+ except KeyError:
+ pass
+ if unicodeCodepoint and unicodeCodepoint not in ignore:
+ result += unichr(unicodeCodepoint)
+ else:
+ # Leave the entity unchanged
+ result += text[match.start():match.end()]
+ i = match.end()
+ else:
+ result += text
+ found = False
+ return result
+
+def url2unicode(title, site, site2 = None):
+ """Convert url-encoded text to unicode using site's encoding.
+
+ If site2 is provided, try its encodings as well. Uses the first encoding
+ that doesn't cause an error.
+
+ """
+ # create a list of all possible encodings for both hint sites
+ encList = [site.encoding()] + list(site.encodings())
+ if site2 and site2 <> site:
+ encList.append(site2.encoding())
+ encList += list(site2.encodings())
+ firstException = None
+ # try to handle all encodings (will probably retry utf-8)
+ for enc in encList:
+ try:
+ t = title.encode(enc)
+ t = urllib.unquote(t)
+ return unicode(t, enc)
+ except UnicodeError, ex:
+ if not firstException:
+ firstException = ex
+ pass
+ # Couldn't convert, raise the original exception
+ raise firstException
+
Property changes on: branches/rewrite/pywikibot/page.py
___________________________________________________________________
Added: svn:keywords
+ Author Date Id Revision
Added: svn:eol-style
+ native
Modified: branches/rewrite/pywikibot/pagegenerators.py
===================================================================
--- branches/rewrite/pywikibot/pagegenerators.py 2008-12-16 19:34:48 UTC (rev 6155)
+++ branches/rewrite/pywikibot/pagegenerators.py 2008-12-16 19:40:20 UTC (rev 6156)
@@ -1,965 +1,965 @@
-# -*- coding: utf-8 -*-
-"""This module offers a wide variety of page generators. A page generator is an
-object that is iterable (see http://www.python.org/dev/peps/pep-0255/ ) and
-that yields page objects on which other scripts can then work.
-
-In general, there is no need to run this script directly. It can, however,
-be run for testing purposes. It will then print the page titles to standard
-output.
-
-These parameters are supported to specify which pages titles to print:
-
-¶ms;
-"""
-#
-# (C) Pywikipedia bot team, 2008
-#
-# Distributed under the terms of the MIT license.
-#
-__version__ = '$Id: $'
-
-import pywikibot
-
-import itertools
-import Queue
-import re
-import sys
-import threading
-
-
-# ported from version 1 for backwards-compatibility
-# most of these functions just wrap a Site or Page method that returns
-# a generator
-
-parameterHelp = """\
--cat Work on all pages which are in a specific category.
- Argument can also be given as "-cat:categoryname" or
- as "-cat:categoryname|fromtitle".
-
--catr Like -cat, but also recursively includes pages in
- subcategories, sub-subcategories etc. of the
- given category.
- Argument can also be given as "-catr:categoryname" or
- as "-catr:categoryname|fromtitle".
-
--subcats Work on all subcategories of a specific category.
- Argument can also be given as "-subcats:categoryname" or
- as "-subcats:categoryname|fromtitle".
-
--subcatsr Like -subcats, but also includes sub-subcategories etc. of
- the given category.
- Argument can also be given as "-subcatsr:categoryname" or
- as "-subcatsr:categoryname|fromtitle".
-
--uncat Work on all pages which are not categorised.
-
--uncatcat Work on all categories which are not categorised.
-
--uncatfiles Work on all files which are not categorised.
-
--file Read a list of pages to treat from the named text file.
- Page titles in the file must be enclosed with [[brackets]].
- Argument can also be given as "-file:filename".
-
--filelinks Work on all pages that use a certain image/media file.
- Argument can also be given as "-filelinks:filename".
-
--yahoo Work on all pages that are found in a Yahoo search.
- Depends on python module pYsearch. See yahoo_appid in
- config.py for instructions.
-
--search Work on all pages that are found in a MediaWiki search
- across all namespaces.
-
--google Work on all pages that are found in a Google search.
- You need a Google Web API license key. Note that Google
- doesn't give out license keys anymore. See google_key in
- config.py for instructions.
- Argument can also be given as "-google:searchstring".
-
--interwiki Work on the given page and all equivalent pages in other
- languages. This can, for example, be used to fight
- multi-site spamming.
- Attention: this will cause the bot to modify
- pages on several wiki sites, this is not well tested,
- so check your edits!
-
--links Work on all pages that are linked from a certain page.
- Argument can also be given as "-links:linkingpagetitle".
-
--new Work on the 60 newest pages. If given as -new:x, will work
- on the x newest pages.
-
--imagelinks Work on all images that are linked from a certain page.
- Argument can also be given as "-imagelinks:linkingpagetitle".
-
--newimages Work on the 100 newest images. If given as -newimages:x,
- will work on the x newest images.
-
--ref Work on all pages that link to a certain page.
- Argument can also be given as "-ref:referredpagetitle".
-
--start Specifies that the robot should go alphabetically through
- all pages on the home wiki, starting at the named page.
- Argument can also be given as "-start:pagetitle".
-
- You can also include a namespace. For example,
- "-start:Template:!" will make the bot work on all pages
- in the template namespace.
-
--prefixindex Work on pages commencing with a common prefix.
-
--regex Obsolete, use -titleregex
-
--titleregex Work on titles that match the given regular expression.
-
--transcludes Work on all pages that use a certain template.
- Argument can also be given as "-transcludes:Template:Title".
-
--unusedfiles Work on all description pages of images/media files that are
- not used anywhere.
- Argument can be given as "-unusedfiles:n" where
- n is the maximum number of articles to work on.
-
--unwatched Work on all articles that are not watched by anyone.
- Argument can be given as "-unwatched:n" where
- n is the maximum number of articles to work on.
-
--usercontribs Work on all articles that were edited by a certain user :
- Example : -usercontribs:DumZiBoT
-
--weblink Work on all articles that contain an external link to
- a given URL; may be given as "-weblink:url"
-
--withoutinterwiki Work on all pages that don't have interlanguage links.
- Argument can be given as "-withoutinterwiki:n" where
- n is some number (??).
-"""
-
-docuReplacements = {'¶ms;': parameterHelp}
-
-# if a bot uses GeneratorFactory, the module should include the line
-# docuReplacements = {'¶ms;': pywikibot.pagegenerators.parameterHelp}
-# and include the marker ¶ms; in the module's docstring
-
-
-class GeneratorFactory(object):
- """Process command line arguments and return appropriate page generator."""
-
- def setCategoryGen(self, arg, length, recurse = False):
- if len(arg) == length:
- categoryname = pywikibot.input(u'Please enter the category name:')
- else:
- categoryname = arg[length + 1:]
-
- ind = categoryname.find('|')
- if ind > 0:
- startfrom = categoryname[ind + 1:]
- categoryname = categoryname[:ind]
- else:
- startfrom = None
-
- cat = pywikibot.Category(pywikibot.Link('Category:%s' % categoryname))
- return CategorizedPageGenerator(cat, start=startfrom, recurse=recurse)
-
- def setSubCategoriesGen(self, arg, length, recurse=False):
- if len(arg) == length:
- categoryname = pywikibot.input(u'Please enter the category name:')
- else:
- categoryname = arg[length + 1:]
-
- ind = categoryname.find('|')
- if ind > 0:
- startfrom = categoryname[ind + 1:]
- categoryname = categoryname[:ind]
- else:
- startfrom = None
-
- cat = pywikibot.Category(pywikibot.Link('Category:%s' % categoryname))
- return SubCategoriesPageGenerator(cat, start=startfrom, recurse=recurse)
-
- def handleArg(self, arg):
- gen = None
- if arg.startswith('-filelinks'):
- fileLinksPageTitle = arg[11:]
- if not fileLinksPageTitle:
- fileLinksPageTitle = pywikibot.input(
- u'Links to which image page should be processed?')
- if fileLinksPageTitle.startswith(pywikibot.Site().namespace(6)
- + ":"):
- fileLinksPage = pywikibot.ImagePage(pywikibot.Site(),
- fileLinksPageTitle)
- else:
- fileLinksPage = pywikibot.ImagePage(pywikibot.Site(),
- 'Image:' +
- fileLinksPageTitle)
- gen = FileLinksGenerator(fileLinksPage)
- elif arg.startswith('-unusedfiles'):
- if len(arg) == 12:
- gen = UnusedFilesGenerator()
- else:
- gen = UnusedFilesGenerator(number = int(arg[13:]))
- elif arg.startswith('-unwatched'):
- if len(arg) == 10:
- gen = UnwatchedPagesPageGenerator()
- else:
- gen = UnwatchedPagesPageGenerator(number = int(arg[11:]))
- elif arg.startswith('-usercontribs'):
- gen = UserContributionsGenerator(arg[14:])
- elif arg.startswith('-withoutinterwiki'):
- if len(arg) == 17:
- gen = WithoutInterwikiPageGenerator()
- else:
- gen = WithoutInterwikiPageGenerator(number = int(arg[18:]))
- elif arg.startswith('-interwiki'):
- title = arg[11:]
- if not title:
- title = pywikibot.input(u'Which page should be processed?')
- page = pywikibot.Page(pywikibot.Site(), title)
- gen = InterwikiPageGenerator(page)
- elif arg.startswith('-file'):
- textfilename = arg[6:]
- if not textfilename:
- textfilename = pywikibot.input(
- u'Please enter the local file name:')
- gen = TextfilePageGenerator(textfilename)
- elif arg.startswith('-catr'):
- gen = self.setCategoryGen(arg, 5, recurse = True)
- elif arg.startswith('-cat'):
- gen = self.setCategoryGen(arg, 4)
- elif arg.startswith('-subcatsr'):
- gen = self.setSubCategoriesGen(arg, 9, recurse = True)
- elif arg.startswith('-subcats'):
- gen = self.setSubCategoriesGen(arg, 8)
- elif arg.startswith('-uncatfiles'):
- gen = UnCategorizedImageGenerator()
- elif arg.startswith('-uncatcat'):
- gen = UnCategorizedCategoryGenerator()
- elif arg.startswith('-uncat'):
- gen = UnCategorizedPageGenerator()
- elif arg.startswith('-ref'):
- referredPageTitle = arg[5:]
- if not referredPageTitle:
- referredPageTitle = pywikibot.input(
- u'Links to which page should be processed?')
- referredPage = pywikibot.Page(pywikibot.Site(), referredPageTitle)
- gen = ReferringPageGenerator(referredPage)
- elif arg.startswith('-links'):
- linkingPageTitle = arg[7:]
- if not linkingPageTitle:
- linkingPageTitle = pywikibot.input(
- u'Links from which page should be processed?')
- linkingPage = pywikibot.Page(pywikibot.Site(), linkingPageTitle)
- gen = LinkedPageGenerator(linkingPage)
- elif arg.startswith('-weblink'):
- url = arg[9:]
- if not url:
- url = pywikibot.input(
- u'Pages with which weblink should be processed?')
- gen = LinksearchPageGenerator(url)
- elif arg.startswith('-transcludes'):
- transclusionPageTitle = arg[len('-transcludes:'):]
- if not transclusionPageTitle:
- transclusionPageTitle = pywikibot.input(
- u'Pages that transclude which page should be processed?')
- transclusionPage = pywikibot.Page(pywikibot.Site(),
- 'Template:%s' % transclusionPageTitle)
- gen = ReferringPageGenerator(transclusionPage,
- onlyTemplateInclusion=True)
- elif arg.startswith('-start'):
- if arg.startswith('-startxml'):
- pywikibot.output(u'-startxml : wrong parameter')
- raise ValueError
- firstPageTitle = arg[7:]
- if not firstPageTitle:
- firstPageTitle = pywikibot.input(
- u'At which page do you want to start?')
- namespace = pywikibot.Page(pywikibot.Site(),
- firstPageTitle).namespace()
- firstPageTitle = pywikibot.Page(pywikibot.link(firstPageTitle)
- ).titleWithoutNamespace()
- gen = AllpagesPageGenerator(firstPageTitle, namespace,
- includeredirects=False)
- elif arg.startswith('-prefixindex'):
- prefix = arg[13:]
- namespace = None
- if not prefix:
- prefix = pywikibot.input(
- u'What page names are you looking for?')
- gen = PrefixingPageGenerator(prefix=prefix)
- elif arg.startswith('-newimages'):
- limit = arg[11:] or pywikibot.input(
- u'How many images do you want to load?')
- gen = NewimagesPageGenerator(number=int(limit))
- elif arg.startswith('-new'):
- if len(arg) >=5:
- gen = NewpagesPageGenerator(number=int(arg[5:]))
- else:
- gen = NewpagesPageGenerator(number=60)
- elif arg.startswith('-imagelinks'):
- imagelinkstitle = arg[len('-imagelinks:'):]
- if not imagelinkstitle:
- imagelinkstitle = pywikibot.input(
- u'Images on which page should be processed?')
- imagelinksPage = pywikibot.Page(pywikibot.Link(imagelinkstitle))
- gen = ImagesPageGenerator(imagelinksPage)
- elif arg.startswith('-search'):
- mediawikiQuery = arg[8:]
- if not mediawikiQuery:
- mediawikiQuery = pywikibot.input(
- u'What do you want to search for?')
- # In order to be useful, all namespaces are required
- gen = SearchPageGenerator(mediawikiQuery, namespaces = [])
- elif arg.startswith('-google'):
- gen = GoogleSearchPageGenerator(arg[8:])
- elif arg.startswith('-titleregex'):
- if len(arg) == 6:
- regex = pywikibot.input(
- u'What page names are you looking for?')
- else:
- regex = arg[7:]
- gen = RegexFilterPageGenerator(pywikibot.Site().allpages(), regex)
- elif arg.startswith('-yahoo'):
- gen = YahooSearchPageGenerator(arg[7:])
- else:
- return None
- # make sure all yielded pages are unique
- gen = DuplicateFilterPageGenerator(gen)
- return gen
-
-
-class ThreadedGenerator(threading.Thread):
- """Look-ahead generator class.
-
- Runs a generator in a separate thread and queues the results; can
- be called like a regular generator.
-
- Subclasses should override self.generator, _not_ self.run
-
- Important: the generator thread will stop itself if the generator's
- internal queue is exhausted; but, if the calling program does not use
- all the generated values, it must call the generator's stop() method to
- stop the background thread. Example usage:
-
- >>> gen = ThreadedGenerator(target=foo)
- >>> try:
- ... for data in gen:
- ... do_work(data)
- ... finally:
- ... gen.stop()
-
- """ #NOT CURRENTLY USED: Intended for future development
-
- def __init__(self, group=None, target=None, name="GeneratorThread",
- args=(), kwargs=None, qsize=65536):
- """Constructor. Takes same keyword arguments as threading.Thread.
-
- target must be a generator function (or other callable that returns
- an iterable object).
-
- @param qsize: The size of the lookahead queue. The larger the qsize,
- the more values will be computed in advance of use (which can eat
- up memory and processor time).
- @type qsize: int
-
- """
- if kwargs is None:
- kwargs = {}
- if target:
- self.generator = target
- if not hasattr(self, "generator"):
- raise RuntimeError("No generator for ThreadedGenerator to run.")
- self.args, self.kwargs = args, kwargs
- threading.Thread.__init__(self, group=group, name=name)
- self.queue = Queue.Queue(qsize)
- self.finished = threading.Event()
-
- def __iter__(self):
- """Iterate results from the queue."""
- if not self.isAlive() and not self.finished.isSet():
- self.start()
- # if there is an item in the queue, yield it, otherwise wait
- while not self.finished.isSet():
- try:
- yield self.queue.get(True, 0.25)
- except Queue.Empty:
- pass
- except KeyboardInterrupt:
- self.stop()
-
- def stop(self):
- """Stop the background thread."""
-## if not self.finished.isSet():
-## pywikibot.output("DEBUG: signalling %s to stop." % self)
- self.finished.set()
-
- def run(self):
- """Run the generator and store the results on the queue."""
- self.__gen = self.generator(*self.args, **self.kwargs)
- for result in self.__gen:
- while True:
- if self.finished.isSet():
-## pywikibot.output("DEBUG: %s received stop signal." % self)
- return
- try:
- self.queue.put_nowait(result)
- except Queue.Full:
- time.sleep(0.25)
- continue
- break
- # wait for queue to be emptied, then kill the thread
- while not self.finished.isSet() and not self.queue.empty():
- time.sleep(0.25)
- self.stop()
-## pywikibot.output("DEBUG: %s stopped because generator exhausted." % self)
-
-
-def AllpagesPageGenerator(start ='!', namespace=None, includeredirects=True,
- site=None):
- """
- Using the Allpages special page, retrieve all articles' titles, and yield
- page objects.
- If includeredirects is False, redirects are not included. If
- includeredirects equals the string 'only', only redirects are added.
- """
- if site is None:
- site = pywikibot.getSite()
- if includeredirects:
- if includeredirects == 'only':
- filterredir = True
- else:
- filterredir = None
- else:
- filterredir = False
- return site.allpages(start=start, namespace=namespace,
- filterredir=filterredir)
-
-
-def PrefixingPageGenerator(prefix, namespace=None, includeredirects=True,
- site=None):
- if site is None:
- site = pywikibot.Site()
- page = pywikibot.Page(site, prefix)
- if namespace is None:
- namespace = page.namespace()
- title = page.titleWithoutNamespace()
- if includeredirects:
- if includeredirects == 'only':
- filterredir = True
- else:
- filterredir = None
- else:
- filterredir = False
- return site.allpages(prefix=title, namespace=namespace,
- filterredir=filterredir)
-
-
-def NewpagesPageGenerator(number=100, get_redirect=False, repeat=False,
- site=None):
- # API does not (yet) have a newpages function, so this tries to duplicate
- # it by filtering the recentchanges output
- # defaults to namespace 0 because that's how Special:Newpages defaults
- if site is None:
- site = pywikibot.Site()
- return site.recentchanges(limit=number, showredirects=get_redirect,
- changetype="new", namespaces=0)
-
-
-def FileLinksGenerator(referredImagePage):
- return referredImagePage.usingPages()
-
-
-def ImagesPageGenerator(pageWithImages):
- return pageWithImages.imagelinks()
-
-
-def InterwikiPageGenerator(page):
- """Iterator over all interwiki (non-language) links on a page."""
- for link in page.interwiki():
- yield pywikibot.Page(link)
-
-
-def LanguageLinksPageGenerator(page):
- """Iterator over all interwiki language links on a page."""
- for link in page.langlinks():
- yield pywikibot.Page(link)
-
-
-def ReferringPageGenerator(referredPage, followRedirects=False,
- withTemplateInclusion=True,
- onlyTemplateInclusion=False):
- '''Yields all pages referring to a specific page.'''
- return referredPage.getReferences(
- follow_redirects=followRedirects,
- withTemplateInclusion=withTemplateInclusion,
- onlyTemplateInclusion=onlyTemplateInclusion)
-
-
-def CategorizedPageGenerator(category, recurse=False, start=None):
- '''Yield all pages in a specific category.
-
- If recurse is True, pages in subcategories are included as well; if
- recurse is an int, only subcategories to that depth will be included
- (e.g., recurse=2 will get pages in subcats and sub-subcats, but will
- not go any further).
- If start is a string value, only pages whose sortkey comes after start
- alphabetically are included.
-
- ''' # TODO: page generator could be modified to use cmstartsortkey ...
- for a in category.articles(recurse=recurse):
- if start is None or a.title(withNamespace=False) >= start:
- yield a
-
-
-def SubCategoriesPageGenerator(category, recurse=False, start=None):
- '''Yields all subcategories in a specific category.
-
- If recurse is True, pages in subcategories are included as well; if
- recurse is an int, only subcategories to that depth will be included
- (e.g., recurse=2 will get pages in subcats and sub-subcats, but will
- not go any further).
- If start is a string value, only categories whose sortkey comes after
- start alphabetically are included.
-
- ''' # TODO: page generator could be modified to use cmstartsortkey ...
- for s in category.subcategories(recurse=recurse):
- if start is None or s.title(withNamespace=False) >= start:
- yield s
-
-
-def LinkedPageGenerator(linkingPage):
- """Yields all pages linked from a specific page."""
- return linkingPage.linkedPages()
-
-
-def TextfilePageGenerator(filename=None, site=None):
- """Iterate pages from a list in a text file.
-
- The file must contain page links between double-square-brackets. The
- generator will yield each corresponding Page object.
-
- @param filename: the name of the file that should be read. If no name is
- given, the generator prompts the user.
- @param site: the default Site for which Page objects should be created
-
- """
- if filename is None:
- filename = pywikibot.input(u'Please enter the filename:')
- if site is None:
- site = pywikibot.Site()
- f = codecs.open(filename, 'r', config.textfile_encoding)
- for linkmatch in pywikibot.link_regex.finditer(f.read()):
- # If the link is in interwiki format, the Page object may reside
- # on a different Site than the default.
- # This makes it possible to work on different wikis using a single
- # text file, but also could be dangerous because you might
- # inadvertently change pages on another wiki!
- yield pywikibot.Page(pywikibot.Link(linkmatch.groups("title"), site))
- f.close()
-
-
-def PagesFromTitlesGenerator(iterable, site=None):
- """Generate pages from the titles (unicode strings) yielded by iterable."""
- if site is None:
- site = pywikibot.Site()
- for title in iterable:
- if not isinstance(title, basestring):
- break
- yield pywikibot.Page(pywikibot.Link(title, site))
-
-
-def UserContributionsGenerator(username, number=250, namespaces=None,
- site=None):
- """Yields number unique pages edited by user:username
- namespaces : list of namespace numbers to fetch contribs from
-
- """
- if site is None:
- site = pywikibot.Site()
- return site.usercontribs(user=username, limit=number, namespaces=namespaces)
-
-
-def NamespaceFilterPageGenerator(generator, namespaces, site=None):
- """
- Wraps around another generator. Yields only those pages that are in one
- of the given namespaces.
-
- The namespace list can contain both integers (namespace numbers) and
- strings/unicode strings (namespace names).
-
- """
- if site is None:
- site = pywikibot.Site()
- # convert namespace names to namespace numbers
- for i in xrange(len(namespaces)):
- ns = namespaces[i]
- if isinstance(ns, basestring):
- index = site.getNamespaceIndex(ns)
- if index is None:
- raise ValueError(u'Unknown namespace: %s' % ns)
- namespaces[i] = index
- for page in generator:
- if page.namespace() in namespaces:
- yield page
-
-
-def RedirectFilterPageGenerator(generator):
- """Yields pages from another generator that are not redirects."""
- for page in generator:
- if not page.isRedirectPage():
- yield page
-
-
-def DuplicateFilterPageGenerator(generator):
- """Yield all unique pages from another generator, omitting duplicates."""
- seenPages = {}
- for page in generator:
- if page not in seenPages:
- seenPages[page] = None
- yield page
-
-
-def RegexFilterPageGenerator(generator, regex):
- """Yield pages from another generator whose titles match regex."""
- reg = re.compile(regex, re.I)
- for page in generator:
- if reg.match(page.titleWithoutNamespace()):
- yield page
-
-
-def CombinedPageGenerator(generators):
- return itertools.chain(*generators)
-
-
-def CategoryGenerator(generator):
- """Yield pages from another generator as Category objects.
-
- Makes sense only if it is ascertained that only categories are being
- retrieved.
-
- """
- for page in generator:
- yield pywikibot.Category(page)
-
-
-def PageWithTalkPageGenerator(generator):
- """
- Wraps around another generator. Yields the same pages, but for non-talk
- pages, it also includes associated talk pages.
- This generator does not check if the talk page in fact exists.
- """
- for page in generator:
- yield page
- if not page.isTalkPage():
- yield page.toggleTalkPage()
-
-
-def PreloadingGenerator(generator, pageNumber=60, lookahead=10):
- """Yield preloaded pages taken from another generator."""
-
- # pages may be on more than one site, for example if an interwiki
- # generator is used, so use a separate preloader for each site
- sites = {}
- # build a list of pages for each site found in the iterator
- for page in generator:
- sites.setdefault(page.site(), []).append(page)
- return itertools.chain(*(site.preloadpages(sites[site], pageNumber)
- for site in sites))
-
-
-#TODO below
-
-def UnusedFilesGenerator(number=100, repeat=False, site=None, extension=None):
- if site is None:
- site = pywikibot.Site()
- for page in site.unusedfiles(number=number, repeat=repeat,
- extension=extension):
- yield pywikibot.ImagePage(page.site(), page.title())
-
-def WithoutInterwikiPageGenerator(number=100, repeat=False, site=None):
- if site is None:
- site = pywikibot.Site()
- for page in site.withoutinterwiki(number=number, repeat=repeat):
- yield page
-
-def UnCategorizedCategoryGenerator(number = 100, repeat = False, site = None):
- if site is None:
- site = pywikibot.Site()
- for page in site.uncategorizedcategories(number=number, repeat=repeat):
- yield page
-
-def UnCategorizedImageGenerator(number = 100, repeat = False, site = None):
- if site is None:
- site = pywikibot.Site()
- for page in site.uncategorizedimages(number=number, repeat=repeat):
- yield page
-
-def NewimagesPageGenerator(number = 100, repeat = False, site = None):
- if site is None:
- site = pywikibot.Site()
- for page in site.newimages(number, repeat=repeat):
- yield page[0]
-
-def UnCategorizedPageGenerator(number = 100, repeat = False, site = None):
- if site is None:
- site = pywikibot.Site()
- for page in site.uncategorizedpages(number=number, repeat=repeat):
- yield page
-
-def LonelyPagesPageGenerator(number = 100, repeat = False, site = None):
- if site is None:
- site = pywikibot.Site()
- for page in site.lonelypages(number=number, repeat=repeat):
- yield page
-
-def UnwatchedPagesPageGenerator(number = 100, repeat = False, site = None):
- if site is None:
- site = pywikibot.Site()
- for page in site.unwatchedpages(number=number, repeat=repeat):
- yield page
-
-def AncientPagesPageGenerator(number = 100, repeat = False, site = None):
- if site is None:
- site = pywikibot.Site()
- for page in site.ancientpages(number=number, repeat=repeat):
- yield page[0]
-
-def DeadendPagesPageGenerator(number = 100, repeat = False, site = None):
- if site is None:
- site = pywikibot.Site()
- for page in site.deadendpages(number=number, repeat=repeat):
- yield page
-
-def LongPagesPageGenerator(number = 100, repeat = False, site = None):
- if site is None:
- site = pywikibot.Site()
- for page in site.longpages(number=number, repeat=repeat):
- yield page[0]
-
-def ShortPagesPageGenerator(number = 100, repeat = False, site = None):
- if site is None:
- site = pywikibot.Site()
- for page in site.shortpages(number=number, repeat=repeat):
- yield page[0]
-
-def LinksearchPageGenerator(link, step=500, site=None):
- """Yields all pages that include a specified link, according to
- [[Special:Linksearch]].
-
- """
- if site is None:
- site = pywikibot.Site()
- for page in site.linksearch(link, limit=step):
- yield page
-
-def SearchPageGenerator(query, number = 100, namespaces = None, site = None):
- """
- Provides a list of results using the internal MediaWiki search engine
- """
- if site is None:
- site = pywikibot.Site()
- for page in site.search(query, number=number, namespaces = namespaces):
- yield page[0]
-
-class YahooSearchPageGenerator:
- '''
- To use this generator, install pYsearch
- '''
- def __init__(self, query = None, count = 100, site = None): # values larger than 100 fail
- self.query = query or pywikibot.input(u'Please enter the search query:')
- self.count = count
- if site is None:
- site = pywikibot.Site()
- self.site = site
-
- def queryYahoo(self, query):
- from yahoo.search.web import WebSearch
- srch = WebSearch(config.yahoo_appid, query=query, results=self.count)
-
- dom = srch.get_results()
- results = srch.parse_results(dom)
- for res in results:
- url = res.Url
- yield url
-
- def __iter__(self):
- # restrict query to local site
- localQuery = '%s site:%s' % (self.query, self.site.hostname())
- base = 'http://%s%s' % (self.site.hostname(), self.site.nice_get_address(''))
- for url in self.queryYahoo(localQuery):
- if url[:len(base)] == base:
- title = url[len(base):]
- page = pywikibot.Page(self.site, title)
- yield page
-
-class GoogleSearchPageGenerator:
- '''
- To use this generator, you must install the pyGoogle module from
- http://pygoogle.sf.net/ and get a Google Web API license key from
- http://www.google.com/apis/index.html . The google_key must be set to your
- license key in your configuration.
- '''
- def __init__(self, query = None, site = None):
- self.query = query or pywikibot.input(u'Please enter the search query:')
- if site is None:
- site = pywikibot.Site()
- self.site = site
-
- #########
- # partially commented out because it is probably not in compliance with Google's "Terms of
- # service" (see 5.3, http://www.google.com/accounts/TOS?loc=US)
- def queryGoogle(self, query):
- #if config.google_key:
- if True:
- #try:
- for url in self.queryViaSoapApi(query):
- yield url
- return
- #except ImportError:
- #pass
- # No google license key, or pygoogle not installed. Do it the ugly way.
- #for url in self.queryViaWeb(query):
- # yield url
-
- def queryViaSoapApi(self, query):
- import google
- google.LICENSE_KEY = config.google_key
- offset = 0
- estimatedTotalResultsCount = None
- while not estimatedTotalResultsCount or offset < estimatedTotalResultsCount:
- while (True):
- # Google often yields 502 errors.
- try:
- pywikibot.output(u'Querying Google, offset %i' % offset)
- data = google.doGoogleSearch(query, start = offset, filter = False)
- break
- except KeyboardInterrupt:
- raise
- except:
- # SOAPpy.Errors.HTTPError or SOAP.HTTPError (502 Bad Gateway)
- # can happen here, depending on the module used. It's not easy
- # to catch this properly because pygoogle decides which one of
- # the soap modules to use.
- pywikibot.output(u"An error occured. Retrying in 10 seconds...")
- time.sleep(10)
- continue
-
- for result in data.results:
- #print 'DBG: ', result.URL
- yield result.URL
- # give an estimate of pages to work on, but only once.
- if not estimatedTotalResultsCount:
- pywikibot.output(u'Estimated total result count: %i pages.' % data.meta.estimatedTotalResultsCount)
- estimatedTotalResultsCount = data.meta.estimatedTotalResultsCount
- #print 'estimatedTotalResultsCount: ', estimatedTotalResultsCount
- offset += 10
-
- #########
- # commented out because it is probably not in compliance with Google's "Terms of
- # service" (see 5.3, http://www.google.com/accounts/TOS?loc=US)
-
- #def queryViaWeb(self, query):
- #"""
- #Google has stopped giving out API license keys, and sooner or later
- #they will probably shut down the service.
- #This is a quick and ugly solution: we just grab the search results from
- #the normal web interface.
- #"""
- #linkR = re.compile(r'<a href="([^>"]+?)" class=l>', re.IGNORECASE)
- #offset = 0
-
- #while True:
- #pywikibot.output("Google: Querying page %d" % (offset / 100 + 1))
- #address = "http://www.google.com/search?q=%s&num=100&hl=en&start=%d" % (urllib.quote_plus(query), offset)
- ## we fake being Firefox because Google blocks unknown browsers
- #request = urllib2.Request(address, None, {'User-Agent': 'Mozilla/5.0 (X11; U; Linux i686; de; rv:1.8) Gecko/20051128 SUSE/1.5-0.1 Firefox/1.5'})
- #urlfile = urllib2.urlopen(request)
- #page = urlfile.read()
- #urlfile.close()
- #for url in linkR.findall(page):
- #yield url
- #if "<div id=nn>" in page: # Is there a "Next" link for next page of results?
- #offset += 100 # Yes, go to next page of results.
- #else:
- #return
- #########
-
- def __iter__(self):
- # restrict query to local site
- localQuery = '%s site:%s' % (self.query, self.site.hostname())
- base = 'http://%s%s' % (self.site.hostname(), self.site.nice_get_address(''))
- for url in self.queryGoogle(localQuery):
- if url[:len(base)] == base:
- title = url[len(base):]
- page = pywikibot.Page(self.site, title)
- # Google contains links in the format http://de.wikipedia.org/wiki/en:Foobar
- if page.site() == self.site:
- yield page
-
-def MySQLPageGenerator(query, site = None):
- import MySQLdb as mysqldb
- if site is None:
- site = pywikibot.Site()
- conn = mysqldb.connect(config.db_hostname, db = site.dbName(),
- user = config.db_username,
- passwd = config.db_password)
- cursor = conn.cursor()
- pywikibot.output(u'Executing query:\n%s' % query)
- query = query.encode(site.encoding())
- cursor.execute(query)
- while True:
- try:
- namespaceNumber, pageName = cursor.fetchone()
- print namespaceNumber, pageName
- except TypeError:
- # Limit reached or no more results
- break
- #print pageName
- if pageName:
- namespace = site.namespace(namespaceNumber)
- pageName = unicode(pageName, site.encoding())
- if namespace:
- pageTitle = '%s:%s' % (namespace, pageName)
- else:
- pageTitle = pageName
- page = pywikibot.Page(site, pageTitle)
- yield page
-
-def YearPageGenerator(start = 1, end = 2050, site = None):
- if site is None:
- site = pywikibot.Site()
- pywikibot.output(u"Starting with year %i" % start)
- for i in xrange(start, end + 1):
- if i % 100 == 0:
- pywikibot.output(u'Preparing %i...' % i)
- # There is no year 0
- if i != 0:
- current_year = date.formatYear(site.lang, i )
- yield pywikibot.Page(site, current_year)
-
-def DayPageGenerator(startMonth = 1, endMonth = 12, site = None):
- if site is None:
- site = pywikibot.Site()
- fd = date.FormatDate(site)
- firstPage = pywikibot.Page(site, fd(startMonth, 1))
- pywikibot.output(u"Starting with %s" % firstPage.aslink())
- for month in xrange(startMonth, endMonth+1):
- for day in xrange(1, date.getNumberOfDaysInMonth(month)+1):
- yield pywikibot.Page(site, fd(month, day))
-
-
-if __name__ == "__main__":
- try:
- gen = None
- genFactory = GeneratorFactory()
- for arg in pywikibot.handleArgs():
- generator = genFactory.handleArg(arg)
- if generator:
- gen = generator
- if gen:
- for page in gen:
- pywikibot.output(page.title(), toStdout = True)
- else:
- pywikibot.showHelp()
- finally:
- pywikibot.stopme()
+# -*- coding: utf-8 -*-
+"""This module offers a wide variety of page generators. A page generator is an
+object that is iterable (see http://www.python.org/dev/peps/pep-0255/ ) and
+that yields page objects on which other scripts can then work.
+
+In general, there is no need to run this script directly. It can, however,
+be run for testing purposes. It will then print the page titles to standard
+output.
+
+These parameters are supported to specify which pages titles to print:
+
+¶ms;
+"""
+#
+# (C) Pywikipedia bot team, 2008
+#
+# Distributed under the terms of the MIT license.
+#
+__version__ = '$Id$'
+
+import pywikibot
+
+import itertools
+import Queue
+import re
+import sys
+import threading
+
+
+# ported from version 1 for backwards-compatibility
+# most of these functions just wrap a Site or Page method that returns
+# a generator
+
+parameterHelp = """\
+-cat Work on all pages which are in a specific category.
+ Argument can also be given as "-cat:categoryname" or
+ as "-cat:categoryname|fromtitle".
+
+-catr Like -cat, but also recursively includes pages in
+ subcategories, sub-subcategories etc. of the
+ given category.
+ Argument can also be given as "-catr:categoryname" or
+ as "-catr:categoryname|fromtitle".
+
+-subcats Work on all subcategories of a specific category.
+ Argument can also be given as "-subcats:categoryname" or
+ as "-subcats:categoryname|fromtitle".
+
+-subcatsr Like -subcats, but also includes sub-subcategories etc. of
+ the given category.
+ Argument can also be given as "-subcatsr:categoryname" or
+ as "-subcatsr:categoryname|fromtitle".
+
+-uncat Work on all pages which are not categorised.
+
+-uncatcat Work on all categories which are not categorised.
+
+-uncatfiles Work on all files which are not categorised.
+
+-file Read a list of pages to treat from the named text file.
+ Page titles in the file must be enclosed with [[brackets]].
+ Argument can also be given as "-file:filename".
+
+-filelinks Work on all pages that use a certain image/media file.
+ Argument can also be given as "-filelinks:filename".
+
+-yahoo Work on all pages that are found in a Yahoo search.
+ Depends on python module pYsearch. See yahoo_appid in
+ config.py for instructions.
+
+-search Work on all pages that are found in a MediaWiki search
+ across all namespaces.
+
+-google Work on all pages that are found in a Google search.
+ You need a Google Web API license key. Note that Google
+ doesn't give out license keys anymore. See google_key in
+ config.py for instructions.
+ Argument can also be given as "-google:searchstring".
+
+-interwiki Work on the given page and all equivalent pages in other
+ languages. This can, for example, be used to fight
+ multi-site spamming.
+ Attention: this will cause the bot to modify
+ pages on several wiki sites, this is not well tested,
+ so check your edits!
+
+-links Work on all pages that are linked from a certain page.
+ Argument can also be given as "-links:linkingpagetitle".
+
+-new Work on the 60 newest pages. If given as -new:x, will work
+ on the x newest pages.
+
+-imagelinks Work on all images that are linked from a certain page.
+ Argument can also be given as "-imagelinks:linkingpagetitle".
+
+-newimages Work on the 100 newest images. If given as -newimages:x,
+ will work on the x newest images.
+
+-ref Work on all pages that link to a certain page.
+ Argument can also be given as "-ref:referredpagetitle".
+
+-start Specifies that the robot should go alphabetically through
+ all pages on the home wiki, starting at the named page.
+ Argument can also be given as "-start:pagetitle".
+
+ You can also include a namespace. For example,
+ "-start:Template:!" will make the bot work on all pages
+ in the template namespace.
+
+-prefixindex Work on pages commencing with a common prefix.
+
+-regex Obsolete, use -titleregex
+
+-titleregex Work on titles that match the given regular expression.
+
+-transcludes Work on all pages that use a certain template.
+ Argument can also be given as "-transcludes:Template:Title".
+
+-unusedfiles Work on all description pages of images/media files that are
+ not used anywhere.
+ Argument can be given as "-unusedfiles:n" where
+ n is the maximum number of articles to work on.
+
+-unwatched Work on all articles that are not watched by anyone.
+ Argument can be given as "-unwatched:n" where
+ n is the maximum number of articles to work on.
+
+-usercontribs Work on all articles that were edited by a certain user :
+ Example : -usercontribs:DumZiBoT
+
+-weblink Work on all articles that contain an external link to
+ a given URL; may be given as "-weblink:url"
+
+-withoutinterwiki Work on all pages that don't have interlanguage links.
+ Argument can be given as "-withoutinterwiki:n" where
+ n is some number (??).
+"""
+
+docuReplacements = {'¶ms;': parameterHelp}
+
+# if a bot uses GeneratorFactory, the module should include the line
+# docuReplacements = {'¶ms;': pywikibot.pagegenerators.parameterHelp}
+# and include the marker ¶ms; in the module's docstring
+
+
+class GeneratorFactory(object):
+ """Process command line arguments and return appropriate page generator."""
+
+ def setCategoryGen(self, arg, length, recurse = False):
+ if len(arg) == length:
+ categoryname = pywikibot.input(u'Please enter the category name:')
+ else:
+ categoryname = arg[length + 1:]
+
+ ind = categoryname.find('|')
+ if ind > 0:
+ startfrom = categoryname[ind + 1:]
+ categoryname = categoryname[:ind]
+ else:
+ startfrom = None
+
+ cat = pywikibot.Category(pywikibot.Link('Category:%s' % categoryname))
+ return CategorizedPageGenerator(cat, start=startfrom, recurse=recurse)
+
+ def setSubCategoriesGen(self, arg, length, recurse=False):
+ if len(arg) == length:
+ categoryname = pywikibot.input(u'Please enter the category name:')
+ else:
+ categoryname = arg[length + 1:]
+
+ ind = categoryname.find('|')
+ if ind > 0:
+ startfrom = categoryname[ind + 1:]
+ categoryname = categoryname[:ind]
+ else:
+ startfrom = None
+
+ cat = pywikibot.Category(pywikibot.Link('Category:%s' % categoryname))
+ return SubCategoriesPageGenerator(cat, start=startfrom, recurse=recurse)
+
+ def handleArg(self, arg):
+ gen = None
+ if arg.startswith('-filelinks'):
+ fileLinksPageTitle = arg[11:]
+ if not fileLinksPageTitle:
+ fileLinksPageTitle = pywikibot.input(
+ u'Links to which image page should be processed?')
+ if fileLinksPageTitle.startswith(pywikibot.Site().namespace(6)
+ + ":"):
+ fileLinksPage = pywikibot.ImagePage(pywikibot.Site(),
+ fileLinksPageTitle)
+ else:
+ fileLinksPage = pywikibot.ImagePage(pywikibot.Site(),
+ 'Image:' +
+ fileLinksPageTitle)
+ gen = FileLinksGenerator(fileLinksPage)
+ elif arg.startswith('-unusedfiles'):
+ if len(arg) == 12:
+ gen = UnusedFilesGenerator()
+ else:
+ gen = UnusedFilesGenerator(number = int(arg[13:]))
+ elif arg.startswith('-unwatched'):
+ if len(arg) == 10:
+ gen = UnwatchedPagesPageGenerator()
+ else:
+ gen = UnwatchedPagesPageGenerator(number = int(arg[11:]))
+ elif arg.startswith('-usercontribs'):
+ gen = UserContributionsGenerator(arg[14:])
+ elif arg.startswith('-withoutinterwiki'):
+ if len(arg) == 17:
+ gen = WithoutInterwikiPageGenerator()
+ else:
+ gen = WithoutInterwikiPageGenerator(number = int(arg[18:]))
+ elif arg.startswith('-interwiki'):
+ title = arg[11:]
+ if not title:
+ title = pywikibot.input(u'Which page should be processed?')
+ page = pywikibot.Page(pywikibot.Site(), title)
+ gen = InterwikiPageGenerator(page)
+ elif arg.startswith('-file'):
+ textfilename = arg[6:]
+ if not textfilename:
+ textfilename = pywikibot.input(
+ u'Please enter the local file name:')
+ gen = TextfilePageGenerator(textfilename)
+ elif arg.startswith('-catr'):
+ gen = self.setCategoryGen(arg, 5, recurse = True)
+ elif arg.startswith('-cat'):
+ gen = self.setCategoryGen(arg, 4)
+ elif arg.startswith('-subcatsr'):
+ gen = self.setSubCategoriesGen(arg, 9, recurse = True)
+ elif arg.startswith('-subcats'):
+ gen = self.setSubCategoriesGen(arg, 8)
+ elif arg.startswith('-uncatfiles'):
+ gen = UnCategorizedImageGenerator()
+ elif arg.startswith('-uncatcat'):
+ gen = UnCategorizedCategoryGenerator()
+ elif arg.startswith('-uncat'):
+ gen = UnCategorizedPageGenerator()
+ elif arg.startswith('-ref'):
+ referredPageTitle = arg[5:]
+ if not referredPageTitle:
+ referredPageTitle = pywikibot.input(
+ u'Links to which page should be processed?')
+ referredPage = pywikibot.Page(pywikibot.Site(), referredPageTitle)
+ gen = ReferringPageGenerator(referredPage)
+ elif arg.startswith('-links'):
+ linkingPageTitle = arg[7:]
+ if not linkingPageTitle:
+ linkingPageTitle = pywikibot.input(
+ u'Links from which page should be processed?')
+ linkingPage = pywikibot.Page(pywikibot.Site(), linkingPageTitle)
+ gen = LinkedPageGenerator(linkingPage)
+ elif arg.startswith('-weblink'):
+ url = arg[9:]
+ if not url:
+ url = pywikibot.input(
+ u'Pages with which weblink should be processed?')
+ gen = LinksearchPageGenerator(url)
+ elif arg.startswith('-transcludes'):
+ transclusionPageTitle = arg[len('-transcludes:'):]
+ if not transclusionPageTitle:
+ transclusionPageTitle = pywikibot.input(
+ u'Pages that transclude which page should be processed?')
+ transclusionPage = pywikibot.Page(pywikibot.Site(),
+ 'Template:%s' % transclusionPageTitle)
+ gen = ReferringPageGenerator(transclusionPage,
+ onlyTemplateInclusion=True)
+ elif arg.startswith('-start'):
+ if arg.startswith('-startxml'):
+ pywikibot.output(u'-startxml : wrong parameter')
+ raise ValueError
+ firstPageTitle = arg[7:]
+ if not firstPageTitle:
+ firstPageTitle = pywikibot.input(
+ u'At which page do you want to start?')
+ namespace = pywikibot.Page(pywikibot.Site(),
+ firstPageTitle).namespace()
+ firstPageTitle = pywikibot.Page(pywikibot.link(firstPageTitle)
+ ).titleWithoutNamespace()
+ gen = AllpagesPageGenerator(firstPageTitle, namespace,
+ includeredirects=False)
+ elif arg.startswith('-prefixindex'):
+ prefix = arg[13:]
+ namespace = None
+ if not prefix:
+ prefix = pywikibot.input(
+ u'What page names are you looking for?')
+ gen = PrefixingPageGenerator(prefix=prefix)
+ elif arg.startswith('-newimages'):
+ limit = arg[11:] or pywikibot.input(
+ u'How many images do you want to load?')
+ gen = NewimagesPageGenerator(number=int(limit))
+ elif arg.startswith('-new'):
+ if len(arg) >=5:
+ gen = NewpagesPageGenerator(number=int(arg[5:]))
+ else:
+ gen = NewpagesPageGenerator(number=60)
+ elif arg.startswith('-imagelinks'):
+ imagelinkstitle = arg[len('-imagelinks:'):]
+ if not imagelinkstitle:
+ imagelinkstitle = pywikibot.input(
+ u'Images on which page should be processed?')
+ imagelinksPage = pywikibot.Page(pywikibot.Link(imagelinkstitle))
+ gen = ImagesPageGenerator(imagelinksPage)
+ elif arg.startswith('-search'):
+ mediawikiQuery = arg[8:]
+ if not mediawikiQuery:
+ mediawikiQuery = pywikibot.input(
+ u'What do you want to search for?')
+ # In order to be useful, all namespaces are required
+ gen = SearchPageGenerator(mediawikiQuery, namespaces = [])
+ elif arg.startswith('-google'):
+ gen = GoogleSearchPageGenerator(arg[8:])
+ elif arg.startswith('-titleregex'):
+ if len(arg) == 6:
+ regex = pywikibot.input(
+ u'What page names are you looking for?')
+ else:
+ regex = arg[7:]
+ gen = RegexFilterPageGenerator(pywikibot.Site().allpages(), regex)
+ elif arg.startswith('-yahoo'):
+ gen = YahooSearchPageGenerator(arg[7:])
+ else:
+ return None
+ # make sure all yielded pages are unique
+ gen = DuplicateFilterPageGenerator(gen)
+ return gen
+
+
+class ThreadedGenerator(threading.Thread):
+ """Look-ahead generator class.
+
+ Runs a generator in a separate thread and queues the results; can
+ be called like a regular generator.
+
+ Subclasses should override self.generator, _not_ self.run
+
+ Important: the generator thread will stop itself if the generator's
+ internal queue is exhausted; but, if the calling program does not use
+ all the generated values, it must call the generator's stop() method to
+ stop the background thread. Example usage:
+
+ >>> gen = ThreadedGenerator(target=foo)
+ >>> try:
+ ... for data in gen:
+ ... do_work(data)
+ ... finally:
+ ... gen.stop()
+
+ """ #NOT CURRENTLY USED: Intended for future development
+
+ def __init__(self, group=None, target=None, name="GeneratorThread",
+ args=(), kwargs=None, qsize=65536):
+ """Constructor. Takes same keyword arguments as threading.Thread.
+
+ target must be a generator function (or other callable that returns
+ an iterable object).
+
+ @param qsize: The size of the lookahead queue. The larger the qsize,
+ the more values will be computed in advance of use (which can eat
+ up memory and processor time).
+ @type qsize: int
+
+ """
+ if kwargs is None:
+ kwargs = {}
+ if target:
+ self.generator = target
+ if not hasattr(self, "generator"):
+ raise RuntimeError("No generator for ThreadedGenerator to run.")
+ self.args, self.kwargs = args, kwargs
+ threading.Thread.__init__(self, group=group, name=name)
+ self.queue = Queue.Queue(qsize)
+ self.finished = threading.Event()
+
+ def __iter__(self):
+ """Iterate results from the queue."""
+ if not self.isAlive() and not self.finished.isSet():
+ self.start()
+ # if there is an item in the queue, yield it, otherwise wait
+ while not self.finished.isSet():
+ try:
+ yield self.queue.get(True, 0.25)
+ except Queue.Empty:
+ pass
+ except KeyboardInterrupt:
+ self.stop()
+
+ def stop(self):
+ """Stop the background thread."""
+## if not self.finished.isSet():
+## pywikibot.output("DEBUG: signalling %s to stop." % self)
+ self.finished.set()
+
+ def run(self):
+ """Run the generator and store the results on the queue."""
+ self.__gen = self.generator(*self.args, **self.kwargs)
+ for result in self.__gen:
+ while True:
+ if self.finished.isSet():
+## pywikibot.output("DEBUG: %s received stop signal." % self)
+ return
+ try:
+ self.queue.put_nowait(result)
+ except Queue.Full:
+ time.sleep(0.25)
+ continue
+ break
+ # wait for queue to be emptied, then kill the thread
+ while not self.finished.isSet() and not self.queue.empty():
+ time.sleep(0.25)
+ self.stop()
+## pywikibot.output("DEBUG: %s stopped because generator exhausted." % self)
+
+
+def AllpagesPageGenerator(start ='!', namespace=None, includeredirects=True,
+ site=None):
+ """
+ Using the Allpages special page, retrieve all articles' titles, and yield
+ page objects.
+ If includeredirects is False, redirects are not included. If
+ includeredirects equals the string 'only', only redirects are added.
+ """
+ if site is None:
+ site = pywikibot.getSite()
+ if includeredirects:
+ if includeredirects == 'only':
+ filterredir = True
+ else:
+ filterredir = None
+ else:
+ filterredir = False
+ return site.allpages(start=start, namespace=namespace,
+ filterredir=filterredir)
+
+
+def PrefixingPageGenerator(prefix, namespace=None, includeredirects=True,
+ site=None):
+ if site is None:
+ site = pywikibot.Site()
+ page = pywikibot.Page(site, prefix)
+ if namespace is None:
+ namespace = page.namespace()
+ title = page.titleWithoutNamespace()
+ if includeredirects:
+ if includeredirects == 'only':
+ filterredir = True
+ else:
+ filterredir = None
+ else:
+ filterredir = False
+ return site.allpages(prefix=title, namespace=namespace,
+ filterredir=filterredir)
+
+
+def NewpagesPageGenerator(number=100, get_redirect=False, repeat=False,
+ site=None):
+ # API does not (yet) have a newpages function, so this tries to duplicate
+ # it by filtering the recentchanges output
+ # defaults to namespace 0 because that's how Special:Newpages defaults
+ if site is None:
+ site = pywikibot.Site()
+ return site.recentchanges(limit=number, showredirects=get_redirect,
+ changetype="new", namespaces=0)
+
+
+def FileLinksGenerator(referredImagePage):
+ return referredImagePage.usingPages()
+
+
+def ImagesPageGenerator(pageWithImages):
+ return pageWithImages.imagelinks()
+
+
+def InterwikiPageGenerator(page):
+ """Iterator over all interwiki (non-language) links on a page."""
+ for link in page.interwiki():
+ yield pywikibot.Page(link)
+
+
+def LanguageLinksPageGenerator(page):
+ """Iterator over all interwiki language links on a page."""
+ for link in page.langlinks():
+ yield pywikibot.Page(link)
+
+
+def ReferringPageGenerator(referredPage, followRedirects=False,
+ withTemplateInclusion=True,
+ onlyTemplateInclusion=False):
+ '''Yields all pages referring to a specific page.'''
+ return referredPage.getReferences(
+ follow_redirects=followRedirects,
+ withTemplateInclusion=withTemplateInclusion,
+ onlyTemplateInclusion=onlyTemplateInclusion)
+
+
+def CategorizedPageGenerator(category, recurse=False, start=None):
+ '''Yield all pages in a specific category.
+
+ If recurse is True, pages in subcategories are included as well; if
+ recurse is an int, only subcategories to that depth will be included
+ (e.g., recurse=2 will get pages in subcats and sub-subcats, but will
+ not go any further).
+ If start is a string value, only pages whose sortkey comes after start
+ alphabetically are included.
+
+ ''' # TODO: page generator could be modified to use cmstartsortkey ...
+ for a in category.articles(recurse=recurse):
+ if start is None or a.title(withNamespace=False) >= start:
+ yield a
+
+
+def SubCategoriesPageGenerator(category, recurse=False, start=None):
+ '''Yields all subcategories in a specific category.
+
+ If recurse is True, pages in subcategories are included as well; if
+ recurse is an int, only subcategories to that depth will be included
+ (e.g., recurse=2 will get pages in subcats and sub-subcats, but will
+ not go any further).
+ If start is a string value, only categories whose sortkey comes after
+ start alphabetically are included.
+
+ ''' # TODO: page generator could be modified to use cmstartsortkey ...
+ for s in category.subcategories(recurse=recurse):
+ if start is None or s.title(withNamespace=False) >= start:
+ yield s
+
+
+def LinkedPageGenerator(linkingPage):
+ """Yields all pages linked from a specific page."""
+ return linkingPage.linkedPages()
+
+
+def TextfilePageGenerator(filename=None, site=None):
+ """Iterate pages from a list in a text file.
+
+ The file must contain page links between double-square-brackets. The
+ generator will yield each corresponding Page object.
+
+ @param filename: the name of the file that should be read. If no name is
+ given, the generator prompts the user.
+ @param site: the default Site for which Page objects should be created
+
+ """
+ if filename is None:
+ filename = pywikibot.input(u'Please enter the filename:')
+ if site is None:
+ site = pywikibot.Site()
+ f = codecs.open(filename, 'r', config.textfile_encoding)
+ for linkmatch in pywikibot.link_regex.finditer(f.read()):
+ # If the link is in interwiki format, the Page object may reside
+ # on a different Site than the default.
+ # This makes it possible to work on different wikis using a single
+ # text file, but also could be dangerous because you might
+ # inadvertently change pages on another wiki!
+ yield pywikibot.Page(pywikibot.Link(linkmatch.groups("title"), site))
+ f.close()
+
+
+def PagesFromTitlesGenerator(iterable, site=None):
+ """Generate pages from the titles (unicode strings) yielded by iterable."""
+ if site is None:
+ site = pywikibot.Site()
+ for title in iterable:
+ if not isinstance(title, basestring):
+ break
+ yield pywikibot.Page(pywikibot.Link(title, site))
+
+
+def UserContributionsGenerator(username, number=250, namespaces=None,
+ site=None):
+ """Yields number unique pages edited by user:username
+ namespaces : list of namespace numbers to fetch contribs from
+
+ """
+ if site is None:
+ site = pywikibot.Site()
+ return site.usercontribs(user=username, limit=number, namespaces=namespaces)
+
+
+def NamespaceFilterPageGenerator(generator, namespaces, site=None):
+ """
+ Wraps around another generator. Yields only those pages that are in one
+ of the given namespaces.
+
+ The namespace list can contain both integers (namespace numbers) and
+ strings/unicode strings (namespace names).
+
+ """
+ if site is None:
+ site = pywikibot.Site()
+ # convert namespace names to namespace numbers
+ for i in xrange(len(namespaces)):
+ ns = namespaces[i]
+ if isinstance(ns, basestring):
+ index = site.getNamespaceIndex(ns)
+ if index is None:
+ raise ValueError(u'Unknown namespace: %s' % ns)
+ namespaces[i] = index
+ for page in generator:
+ if page.namespace() in namespaces:
+ yield page
+
+
+def RedirectFilterPageGenerator(generator):
+ """Yields pages from another generator that are not redirects."""
+ for page in generator:
+ if not page.isRedirectPage():
+ yield page
+
+
+def DuplicateFilterPageGenerator(generator):
+ """Yield all unique pages from another generator, omitting duplicates."""
+ seenPages = {}
+ for page in generator:
+ if page not in seenPages:
+ seenPages[page] = None
+ yield page
+
+
+def RegexFilterPageGenerator(generator, regex):
+ """Yield pages from another generator whose titles match regex."""
+ reg = re.compile(regex, re.I)
+ for page in generator:
+ if reg.match(page.titleWithoutNamespace()):
+ yield page
+
+
+def CombinedPageGenerator(generators):
+ return itertools.chain(*generators)
+
+
+def CategoryGenerator(generator):
+ """Yield pages from another generator as Category objects.
+
+ Makes sense only if it is ascertained that only categories are being
+ retrieved.
+
+ """
+ for page in generator:
+ yield pywikibot.Category(page)
+
+
+def PageWithTalkPageGenerator(generator):
+ """
+ Wraps around another generator. Yields the same pages, but for non-talk
+ pages, it also includes associated talk pages.
+ This generator does not check if the talk page in fact exists.
+ """
+ for page in generator:
+ yield page
+ if not page.isTalkPage():
+ yield page.toggleTalkPage()
+
+
+def PreloadingGenerator(generator, pageNumber=60, lookahead=10):
+ """Yield preloaded pages taken from another generator."""
+
+ # pages may be on more than one site, for example if an interwiki
+ # generator is used, so use a separate preloader for each site
+ sites = {}
+ # build a list of pages for each site found in the iterator
+ for page in generator:
+ sites.setdefault(page.site(), []).append(page)
+ return itertools.chain(*(site.preloadpages(sites[site], pageNumber)
+ for site in sites))
+
+
+#TODO below
+
+def UnusedFilesGenerator(number=100, repeat=False, site=None, extension=None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.unusedfiles(number=number, repeat=repeat,
+ extension=extension):
+ yield pywikibot.ImagePage(page.site(), page.title())
+
+def WithoutInterwikiPageGenerator(number=100, repeat=False, site=None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.withoutinterwiki(number=number, repeat=repeat):
+ yield page
+
+def UnCategorizedCategoryGenerator(number = 100, repeat = False, site = None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.uncategorizedcategories(number=number, repeat=repeat):
+ yield page
+
+def UnCategorizedImageGenerator(number = 100, repeat = False, site = None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.uncategorizedimages(number=number, repeat=repeat):
+ yield page
+
+def NewimagesPageGenerator(number = 100, repeat = False, site = None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.newimages(number, repeat=repeat):
+ yield page[0]
+
+def UnCategorizedPageGenerator(number = 100, repeat = False, site = None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.uncategorizedpages(number=number, repeat=repeat):
+ yield page
+
+def LonelyPagesPageGenerator(number = 100, repeat = False, site = None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.lonelypages(number=number, repeat=repeat):
+ yield page
+
+def UnwatchedPagesPageGenerator(number = 100, repeat = False, site = None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.unwatchedpages(number=number, repeat=repeat):
+ yield page
+
+def AncientPagesPageGenerator(number = 100, repeat = False, site = None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.ancientpages(number=number, repeat=repeat):
+ yield page[0]
+
+def DeadendPagesPageGenerator(number = 100, repeat = False, site = None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.deadendpages(number=number, repeat=repeat):
+ yield page
+
+def LongPagesPageGenerator(number = 100, repeat = False, site = None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.longpages(number=number, repeat=repeat):
+ yield page[0]
+
+def ShortPagesPageGenerator(number = 100, repeat = False, site = None):
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.shortpages(number=number, repeat=repeat):
+ yield page[0]
+
+def LinksearchPageGenerator(link, step=500, site=None):
+ """Yields all pages that include a specified link, according to
+ [[Special:Linksearch]].
+
+ """
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.linksearch(link, limit=step):
+ yield page
+
+def SearchPageGenerator(query, number = 100, namespaces = None, site = None):
+ """
+ Provides a list of results using the internal MediaWiki search engine
+ """
+ if site is None:
+ site = pywikibot.Site()
+ for page in site.search(query, number=number, namespaces = namespaces):
+ yield page[0]
+
+class YahooSearchPageGenerator:
+ '''
+ To use this generator, install pYsearch
+ '''
+ def __init__(self, query = None, count = 100, site = None): # values larger than 100 fail
+ self.query = query or pywikibot.input(u'Please enter the search query:')
+ self.count = count
+ if site is None:
+ site = pywikibot.Site()
+ self.site = site
+
+ def queryYahoo(self, query):
+ from yahoo.search.web import WebSearch
+ srch = WebSearch(config.yahoo_appid, query=query, results=self.count)
+
+ dom = srch.get_results()
+ results = srch.parse_results(dom)
+ for res in results:
+ url = res.Url
+ yield url
+
+ def __iter__(self):
+ # restrict query to local site
+ localQuery = '%s site:%s' % (self.query, self.site.hostname())
+ base = 'http://%s%s' % (self.site.hostname(), self.site.nice_get_address(''))
+ for url in self.queryYahoo(localQuery):
+ if url[:len(base)] == base:
+ title = url[len(base):]
+ page = pywikibot.Page(self.site, title)
+ yield page
+
+class GoogleSearchPageGenerator:
+ '''
+ To use this generator, you must install the pyGoogle module from
+ http://pygoogle.sf.net/ and get a Google Web API license key from
+ http://www.google.com/apis/index.html . The google_key must be set to your
+ license key in your configuration.
+ '''
+ def __init__(self, query = None, site = None):
+ self.query = query or pywikibot.input(u'Please enter the search query:')
+ if site is None:
+ site = pywikibot.Site()
+ self.site = site
+
+ #########
+ # partially commented out because it is probably not in compliance with Google's "Terms of
+ # service" (see 5.3, http://www.google.com/accounts/TOS?loc=US)
+ def queryGoogle(self, query):
+ #if config.google_key:
+ if True:
+ #try:
+ for url in self.queryViaSoapApi(query):
+ yield url
+ return
+ #except ImportError:
+ #pass
+ # No google license key, or pygoogle not installed. Do it the ugly way.
+ #for url in self.queryViaWeb(query):
+ # yield url
+
+ def queryViaSoapApi(self, query):
+ import google
+ google.LICENSE_KEY = config.google_key
+ offset = 0
+ estimatedTotalResultsCount = None
+ while not estimatedTotalResultsCount or offset < estimatedTotalResultsCount:
+ while (True):
+ # Google often yields 502 errors.
+ try:
+ pywikibot.output(u'Querying Google, offset %i' % offset)
+ data = google.doGoogleSearch(query, start = offset, filter = False)
+ break
+ except KeyboardInterrupt:
+ raise
+ except:
+ # SOAPpy.Errors.HTTPError or SOAP.HTTPError (502 Bad Gateway)
+ # can happen here, depending on the module used. It's not easy
+ # to catch this properly because pygoogle decides which one of
+ # the soap modules to use.
+ pywikibot.output(u"An error occured. Retrying in 10 seconds...")
+ time.sleep(10)
+ continue
+
+ for result in data.results:
+ #print 'DBG: ', result.URL
+ yield result.URL
+ # give an estimate of pages to work on, but only once.
+ if not estimatedTotalResultsCount:
+ pywikibot.output(u'Estimated total result count: %i pages.' % data.meta.estimatedTotalResultsCount)
+ estimatedTotalResultsCount = data.meta.estimatedTotalResultsCount
+ #print 'estimatedTotalResultsCount: ', estimatedTotalResultsCount
+ offset += 10
+
+ #########
+ # commented out because it is probably not in compliance with Google's "Terms of
+ # service" (see 5.3, http://www.google.com/accounts/TOS?loc=US)
+
+ #def queryViaWeb(self, query):
+ #"""
+ #Google has stopped giving out API license keys, and sooner or later
+ #they will probably shut down the service.
+ #This is a quick and ugly solution: we just grab the search results from
+ #the normal web interface.
+ #"""
+ #linkR = re.compile(r'<a href="([^>"]+?)" class=l>', re.IGNORECASE)
+ #offset = 0
+
+ #while True:
+ #pywikibot.output("Google: Querying page %d" % (offset / 100 + 1))
+ #address = "http://www.google.com/search?q=%s&num=100&hl=en&start=%d" % (urllib.quote_plus(query), offset)
+ ## we fake being Firefox because Google blocks unknown browsers
+ #request = urllib2.Request(address, None, {'User-Agent': 'Mozilla/5.0 (X11; U; Linux i686; de; rv:1.8) Gecko/20051128 SUSE/1.5-0.1 Firefox/1.5'})
+ #urlfile = urllib2.urlopen(request)
+ #page = urlfile.read()
+ #urlfile.close()
+ #for url in linkR.findall(page):
+ #yield url
+ #if "<div id=nn>" in page: # Is there a "Next" link for next page of results?
+ #offset += 100 # Yes, go to next page of results.
+ #else:
+ #return
+ #########
+
+ def __iter__(self):
+ # restrict query to local site
+ localQuery = '%s site:%s' % (self.query, self.site.hostname())
+ base = 'http://%s%s' % (self.site.hostname(), self.site.nice_get_address(''))
+ for url in self.queryGoogle(localQuery):
+ if url[:len(base)] == base:
+ title = url[len(base):]
+ page = pywikibot.Page(self.site, title)
+ # Google contains links in the format http://de.wikipedia.org/wiki/en:Foobar
+ if page.site() == self.site:
+ yield page
+
+def MySQLPageGenerator(query, site = None):
+ import MySQLdb as mysqldb
+ if site is None:
+ site = pywikibot.Site()
+ conn = mysqldb.connect(config.db_hostname, db = site.dbName(),
+ user = config.db_username,
+ passwd = config.db_password)
+ cursor = conn.cursor()
+ pywikibot.output(u'Executing query:\n%s' % query)
+ query = query.encode(site.encoding())
+ cursor.execute(query)
+ while True:
+ try:
+ namespaceNumber, pageName = cursor.fetchone()
+ print namespaceNumber, pageName
+ except TypeError:
+ # Limit reached or no more results
+ break
+ #print pageName
+ if pageName:
+ namespace = site.namespace(namespaceNumber)
+ pageName = unicode(pageName, site.encoding())
+ if namespace:
+ pageTitle = '%s:%s' % (namespace, pageName)
+ else:
+ pageTitle = pageName
+ page = pywikibot.Page(site, pageTitle)
+ yield page
+
+def YearPageGenerator(start = 1, end = 2050, site = None):
+ if site is None:
+ site = pywikibot.Site()
+ pywikibot.output(u"Starting with year %i" % start)
+ for i in xrange(start, end + 1):
+ if i % 100 == 0:
+ pywikibot.output(u'Preparing %i...' % i)
+ # There is no year 0
+ if i != 0:
+ current_year = date.formatYear(site.lang, i )
+ yield pywikibot.Page(site, current_year)
+
+def DayPageGenerator(startMonth = 1, endMonth = 12, site = None):
+ if site is None:
+ site = pywikibot.Site()
+ fd = date.FormatDate(site)
+ firstPage = pywikibot.Page(site, fd(startMonth, 1))
+ pywikibot.output(u"Starting with %s" % firstPage.aslink())
+ for month in xrange(startMonth, endMonth+1):
+ for day in xrange(1, date.getNumberOfDaysInMonth(month)+1):
+ yield pywikibot.Page(site, fd(month, day))
+
+
+if __name__ == "__main__":
+ try:
+ gen = None
+ genFactory = GeneratorFactory()
+ for arg in pywikibot.handleArgs():
+ generator = genFactory.handleArg(arg)
+ if generator:
+ gen = generator
+ if gen:
+ for page in gen:
+ pywikibot.output(page.title(), toStdout = True)
+ else:
+ pywikibot.showHelp()
+ finally:
+ pywikibot.stopme()
Property changes on: branches/rewrite/pywikibot/pagegenerators.py
___________________________________________________________________
Added: svn:keywords
+ Author Date Id Revision
Added: svn:eol-style
+ native
Modified: branches/rewrite/pywikibot/site.py
===================================================================
--- branches/rewrite/pywikibot/site.py 2008-12-16 19:34:48 UTC (rev 6155)
+++ branches/rewrite/pywikibot/site.py 2008-12-16 19:40:20 UTC (rev 6156)
@@ -1,2861 +1,2861 @@
- # -*- coding: utf-8 -*-
-"""
-Objects representing MediaWiki sites (wikis) and families (groups of wikis
-on the same topic in different languages).
-"""
-#
-# (C) Pywikipedia bot team, 2008
-#
-# Distributed under the terms of the MIT license.
-#
-__version__ = '$Id: $'
-
-import pywikibot
-from pywikibot import deprecate_arg
-from pywikibot import config
-from pywikibot.throttle import Throttle
-from pywikibot.data import api
-from pywikibot.exceptions import *
-
-try:
- from hashlib import md5
-except ImportError:
- from md5 import md5
-import logging
-import os
-import re
-import sys
-import threading
-import urllib
-
-logger = logging.getLogger("wiki")
-
-class PageInUse(pywikibot.Error):
- """Page cannot be reserved for writing due to existing lock."""
-
-
-def Family(fam=None, fatal=True):
- """Import the named family.
-
- @param fam: family name (if omitted, uses the configured default)
- @type fam: str
- @param fatal: if True, the bot will stop running if the given family is
- unknown. If False, it will only raise a ValueError exception.
- @param fatal: bool
- @return: a Family instance configured for the named family.
-
- """
- if fam == None:
- fam = config.family
- try:
- # first try the built-in families
- exec "import pywikibot.families.%s_family as myfamily" % fam
- except ImportError:
- # next see if user has defined a local family module
- try:
- sys.path.append(config.datafilepath('families'))
- exec "import %s_family as myfamily" % fam
- except ImportError:
- if fatal:
- logger.exception(u"""\
-Error importing the %s family. This probably means the family
-does not exist. Also check your configuration file."""
- % fam)
- sys.exit(1)
- else:
- raise Error("Family %s does not exist" % fam)
- return myfamily.Family()
-
-
-class BaseSite(object):
- """Site methods that are independent of the communication interface."""
- # to implement a specific interface, define a Site class that inherits
- # from this
-
- def __init__(self, code, fam=None, user=None, sysop=None):
- """
- @param code: the site's language code
- @type code: str
- @param fam: wiki family name (optional)
- @type fam: str or Family
- @param user: bot user name (optional)
- @type user: str
- @param sysop: sysop account user name (optional)
- @type sysop: str
-
- """
- self.__code = code.lower()
- if isinstance(fam, basestring) or fam is None:
- self.__family = Family(fam, fatal=False)
- else:
- self.__family = fam
-
- # if we got an outdated language code, use the new one instead.
- if self.__family.obsolete.has_key(self.__code):
- if self.__family.obsolete[self.__code] is not None:
- self.__code = self.__family.obsolete[self.__code]
- else:
- # no such language anymore
- raise NoSuchSite("Language %s in family %s is obsolete"
- % (self.__code, self.__family.name))
- if self.__code not in self.languages():
- if self.__code == 'zh-classic' and 'zh-classical' in self.languages():
- self.__code = 'zh-classical'
- # database hack (database is varchar[10] -> zh-classical
- # is cut to zh-classic.
- else:
- raise NoSuchSite("Language %s does not exist in family %s"
- % (self.__code, self.__family.name))
-
- self._username = [user, sysop]
-
- # following are for use with lock_page and unlock_page methods
- self._pagemutex = threading.Lock()
- self._locked_pages = []
-
- @property
- def throttle(self):
- """Return this Site's throttle. Initialize a new one if needed."""
-
- if not hasattr(self, "_throttle"):
- self._throttle = Throttle(self, multiplydelay=True,
- verbosedelay=True)
- try:
- self.login(False)
- except pywikibot.NoUsername:
- pass
- return self._throttle
-
- @property
- def family(self):
- """The Family object for this Site's wiki family."""
-
- return self.__family
-
- @property
- def code(self):
- """The identifying code for this Site."""
-
- return self.__code
-
- @property
- def lang(self):
- """The ISO language code for this Site.
-
- Presumed to be equal to the wiki prefix, but this can be overridden.
-
- """
- return self.__code
-
- def __cmp__(self, other):
- """Perform equality and inequality tests on Site objects."""
-
- if not isinstance(other, BaseSite):
- return 1
- if self.family == other.family:
- return cmp(self.code, other.code)
- return cmp(self.family.name, other.family.name)
-
- def user(self):
- """Return the currently-logged in bot user, or None."""
-
- if self.logged_in(True):
- return self._username[True]
- elif self.logged_in(False):
- return self._username[False]
- return None
-
- def username(self, sysop = False):
- return self._username[sysop]
-
- def __getattr__(self, attr):
- """Calls to methods not defined in this object are passed to Family."""
-
- if hasattr(self.__class__, attr):
- return self.__class__.attr
- try:
- method = getattr(self.family, attr)
- f = lambda *args, **kwargs: \
- method(self.code, *args, **kwargs)
- if hasattr(method, "__doc__"):
- f.__doc__ = method.__doc__
- return f
- except AttributeError:
- raise AttributeError("%s instance has no attribute '%s'"
- % (self.__class__.__name__, attr) )
-
- def sitename(self):
- """Return string representing this Site's name and language."""
-
- return self.family.name+':'+self.code
-
- __str__ = sitename
-
- def __repr__(self):
- return 'Site("%s", "%s")' % (self.code, self.family.name)
-
- def __hash__(self):
- return hash(repr(self))
-
- def linktrail(self):
- """Return regex for trailing chars displayed as part of a link.
-
- Returns a string, not a compiled regular expression object.
-
- This reads from the family file, and ''not'' from
- [[MediaWiki:Linktrail]], because the MW software currently uses a
- built-in linktrail from its message files and ignores the wiki
- value.
-
- """
- return self.family.linktrail(self.code)
-
- def languages(self):
- """Return list of all valid language codes for this site's Family."""
-
- return self.family.langs.keys()
-
- def validLanguageLinks(self):
- """Return list of language codes that can be used in interwiki links."""
-
- nsnames = sum(self.namespaces().values(), [])
- return [l for l in self.languages()
- if l[:1].upper() + l[1:] not in self.namespaces()]
-
- def ns_index(self, namespace):
- """Given a namespace name, return its int index, or None if invalid."""
-
- for ns in self.namespaces():
- if namespace.lower() in [name.lower()
- for name in self.namespaces()[ns]]:
- return ns
- return None
-
- getNamespaceIndex = ns_index # for backwards-compatibility
-
- def namespaces(self):
- """Return dict of valid namespaces on this wiki."""
-
- return self._namespaces
-
- def ns_normalize(self, value):
- """Return canonical local form of namespace name.
-
- @param value: A namespace name
- @type value: unicode
-
- """
- index = self.ns_index(value)
- return self.namespace(index)
-
- normalizeNamespace = ns_normalize # for backwards-compatibility
-
- def redirect(self, default=True):
- """Return the localized redirect tag for the site.
-
- If default is True, falls back to 'REDIRECT' if the site has no
- special redirect tag.
-
- """
- if default:
- return self.family.redirect.get(self.code, [u"REDIRECT"])[0]
- else:
- return self.family.redirect.get(self.code, None)
-
- def lock_page(self, page, block=True):
- """Lock page for writing. Must be called before writing any page.
-
- We don't want different threads trying to write to the same page
- at the same time, even to different sections.
-
- @param page: the page to be locked
- @type page: pywikibot.Page
- @param block: if true, wait until the page is available to be locked;
- otherwise, raise an exception if page can't be locked
-
- """
- self._pagemutex.acquire()
- try:
- while page in self._locked_pages:
- if not block:
- raise PageInUse
- time.sleep(.25)
- self._locked_pages.append(page.title(withSection=False))
- finally:
- self._pagemutex.release()
-
- def unlock_page(self, page):
- """Unlock page. Call as soon as a write operation has completed.
-
- @param page: the page to be locked
- @type page: pywikibot.Page
-
- """
- self._pagemutex.acquire()
- try:
- self._locked_pages.remove(page.title(withSection=False))
- finally:
- self._pagemutex.release()
-
- def disambcategory(self):
- """Return Category in which disambig pages are listed."""
-
- try:
- name = self.namespace(14)+':'+self.family.disambcatname[self.code]
- except KeyError:
- raise Error(u"No disambiguation category name found for %(site)s"
- % {'site': self})
- return pywikibot.Category(pywikibot.Link(name, self))
-
- def linkto(self, title, othersite = None):
- """Return unicode string in the form of a wikilink to 'title'
-
- Use optional Site argument 'othersite' to generate an interwiki link.
-
- """
- logger.debug("Site.linkto() method is deprecated; use pywikibot.Link")
- return pywikibot.Link(title, self).astext(othersite)
-
- def isInterwikiLink(self, s):
- """Return True if s is in the form of an interwiki link.
-
- If a link object constructed using "s" as the link text parses as
- belonging to a different site, this method returns True.
-
- """
- return (pywikibot.Link(s, self).site != self)
-
- def redirectRegex(self):
- """Return a compiled regular expression matching on redirect pages.
-
- Group 1 in the regex match object will be the target title.
-
- """
- #TODO: is this needed, since the API identifies redirects?
- # (maybe, the API can give false positives)
- default = 'REDIRECT'
- try:
- keywords = set(self.family.redirect[self.code])
- keywords.add(default)
- pattern = r'(?:' + '|'.join(keywords) + ')'
- except KeyError:
- # no localized keyword for redirects
- pattern = r'%s' % default
- # A redirect starts with hash (#), followed by a keyword, then
- # arbitrary stuff, then a wikilink. The wikilink may contain
- # a label, although this is not useful.
- return re.compile(r'\s*#%(pattern)s\s*:?\s*\[\[(.+?)(?:\|.*?)?\]\]'
- % locals(),
- re.IGNORECASE | re.UNICODE | re.DOTALL)
-
- # namespace shortcuts for backwards-compatibility
-
- def special_namespace(self):
- return self.namespace(-1)
-
- def image_namespace(self):
- return self.namespace(6)
-
- def mediawiki_namespace(self):
- return self.namespace(8)
-
- def template_namespace(self):
- return self.namespace(10)
-
- def category_namespace(self):
- return self.namespace(14)
-
- def category_namespaces(self):
- return self.namespace(14, all=True)
-
- # site-specific formatting preferences
-
- def category_on_one_line(self):
- """Return True if this site wants all category links on one line."""
-
- return self.code in self.family.category_on_one_line
-
- def interwiki_putfirst(self):
- """Return list of language codes for ordering of interwiki links."""
-
- return self.family.interwiki_putfirst.get(self.code, None)
-
- def interwiki_putfirst_doubled(self, list_of_links):
- # TODO: is this even needed? No family in the framework has this
- # dictionary defined!
- if self.lang in self.family.interwiki_putfirst_doubled:
- if len(list_of_links) >= \
- self.family.interwiki_putfirst_doubled[self.lang][0]:
- links2 = [lang.language() for lang in list_of_links]
- result = []
- for lang in self.family.interwiki_putfirst_doubled[self.lang][1]:
- try:
- result.append(list_of_links[links2.index(lang)])
- except ValueError:
- pass
- return result
- else:
- return False
- else:
- return False
-
- def getSite(self, code):
- """Return Site object for language 'code' in this Family."""
-
- return pywikibot.Site(code=code, fam=self.family, user=self.user)
-
- # deprecated methods for backwards-compatibility
-
- def fam(self):
- """Return Family object for this Site."""
- return self.family
-
- def urlEncode(self, query):
- """DEPRECATED"""
- return urllib.urlencode(query)
-
- def getUrl(self, path, retry=True, sysop=False, data=None,
- compress=True, no_hostname=False, cookie_only=False):
- """DEPRECATED.
-
- Retained for compatibility only. All arguments except path and data
- are ignored.
-
- """
- if data:
- if not isinstance(data, basestring):
- data = urllib.urlencode(data)
- return pywikibot.comms.data.request(self, path, method="PUT",
- body=data)
- else:
- return pywikibot.comms.data.request(self, path)
-
- def postForm(self, address, predata, sysop=False, cookies=None):
- """DEPRECATED"""
- return self.getUrl(address, data=predata)
-
- def postData(self, address, data, contentType=None, sysop=False,
- compress=True, cookies=None):
- """DEPRECATED"""
- return self.getUrl(address, data=data)
-
- # unsupported methods from version 1
-
- def checkCharset(self, charset):
- raise NotImplementedError
- def getToken(self, getalways=True, getagain=False, sysop=False):
- raise NotImplementedError
- def export_address(self):
- raise NotImplementedError
- def move_address(self):
- raise NotImplementedError
- def delete_address(self, s):
- raise NotImplementedError
- def undelete_view_address(self, s, ts=''):
- raise NotImplementedError
- def undelete_address(self):
- raise NotImplementedError
- def protect_address(self, s):
- raise NotImplementedError
- def unprotect_address(self, s):
- raise NotImplementedError
- def put_address(self, s):
- raise NotImplementedError
- def get_address(self, s):
- raise NotImplementedError
- def nice_get_address(self, s):
- raise NotImplementedError
- def edit_address(self, s):
- raise NotImplementedError
- def purge_address(self, s):
- raise NotImplementedError
- def block_address(self):
- raise NotImplementedError
- def unblock_address(self):
- raise NotImplementedError
- def blocksearch_address(self, s):
- raise NotImplementedError
- def linksearch_address(self, s, limit=500, offset=0):
- raise NotImplementedError
- def search_address(self, q, n=50, ns=0):
- raise NotImplementedError
- def allpages_address(self, s, ns = 0):
- raise NotImplementedError
- def log_address(self, n=50, mode = ''):
- raise NotImplementedError
- def newpages_address(self, n=50):
- raise NotImplementedError
- def longpages_address(self, n=500):
- raise NotImplementedError
- def shortpages_address(self, n=500):
- raise NotImplementedError
- def unusedfiles_address(self, n=500):
- raise NotImplementedError
- def categories_address(self, n=500):
- raise NotImplementedError
- def deadendpages_address(self, n=500):
- raise NotImplementedError
- def ancientpages_address(self, n=500):
- raise NotImplementedError
- def lonelypages_address(self, n=500):
- raise NotImplementedError
- def protectedpages_address(self, n=500):
- raise NotImplementedError
- def unwatchedpages_address(self, n=500):
- raise NotImplementedError
- def uncategorizedcategories_address(self, n=500):
- raise NotImplementedError
- def uncategorizedimages_address(self, n=500):
- raise NotImplementedError
- def uncategorizedpages_address(self, n=500):
- raise NotImplementedError
- def unusedcategories_address(self, n=500):
- raise NotImplementedError
- def withoutinterwiki_address(self, n=500):
- raise NotImplementedError
- def references_address(self, s):
- raise NotImplementedError
- def allmessages_address(self):
- raise NotImplementedError
- def upload_address(self):
- raise NotImplementedError
- def double_redirects_address(self, default_limit = True):
- raise NotImplementedError
- def broken_redirects_address(self, default_limit = True):
- raise NotImplementedError
- def login_address(self):
- raise NotImplementedError
- def captcha_image_address(self, id):
- raise NotImplementedError
- def watchlist_address(self):
- raise NotImplementedError
- def contribs_address(self, target, limit=500, offset=''):
- raise NotImplementedError
-
-
-class APISite(BaseSite):
- """API interface to MediaWiki site.
-
- Do not use directly; use pywikibot.Site function.
-
- """
-## Site methods from version 1.0 (as these are implemented in this file,
-## or declared deprecated/obsolete, they will be removed from this list)
-##########
-## cookies: return user's cookies as a string
-##
-## urlEncode: Encode a query to be sent using an http POST request.
-## postForm: Post form data to an address at this site.
-## postData: Post encoded form data to an http address at this site.
-##
-## shared_image_repository: Return tuple of image repositories used by this
-## site.
-## version: Return MediaWiki version string from Family file.
-## versionnumber: Return int identifying the MediaWiki version.
-## live_version: Return version number read from Special:Version.
-## checkCharset(charset): Warn if charset doesn't match family file.
-##
-## linktrail: Return regex for trailing chars displayed as part of a link.
-## disambcategory: Category in which disambiguation pages are listed.
-##
-## Methods that yield Page objects derived from a wiki's Special: pages
-## (note, some methods yield other information in a tuple along with the
-## Pages; see method docs for details) --
-##
-## newpages(): Special:Newpages
-## newimages(): Special:Log&type=upload
-## longpages(): Special:Longpages
-## shortpages(): Special:Shortpages
-## deadendpages(): Special:Deadendpages
-## ancientpages(): Special:Ancientpages
-## lonelypages(): Special:Lonelypages
-## unwatchedpages(): Special:Unwatchedpages (sysop accounts only)
-## uncategorizedcategories(): Special:Uncategorizedcategories (yields
-## Category objects)
-## uncategorizedpages(): Special:Uncategorizedpages
-## uncategorizedimages(): Special:Uncategorizedimages (yields
-## ImagePage objects)
-## unusedcategories(): Special:Unusuedcategories (yields Category)
-## unusedfiles(): Special:Unusedimages (yields ImagePage)
-## withoutinterwiki: Special:Withoutinterwiki
-## linksearch: Special:Linksearch
-
- def __init__(self, code, fam=None, user=None, sysop=None):
- BaseSite.__init__(self, code, fam, user, sysop)
- self._namespaces = {
- # these are the MediaWiki built-in names, which always work
- # localized names are loaded later upon accessing the wiki
- # namespace prefixes are always case-insensitive, but the
- # canonical forms are capitalized
- -2: [u"Media"],
- -1: [u"Special"],
- 0: [u""],
- 1: [u"Talk"],
- 2: [u"User"],
- 3: [u"User talk"],
- 4: [u"Project"],
- 5: [u"Project talk"],
- 6: [u"Image"],
- 7: [u"Image talk"],
- 8: [u"MediaWiki"],
- 9: [u"MediaWiki talk"],
- 10: [u"Template"],
- 11: [u"Template talk"],
- 12: [u"Help"],
- 13: [u"Help talk"],
- 14: [u"Category"],
- 15: [u"Category talk"],
- }
- self.sitelock = threading.Lock()
- self._msgcache = {}
- return
-
-# ANYTHING BELOW THIS POINT IS NOT YET IMPLEMENTED IN __init__()
- self.nocapitalize = self.__code in self.family.nocapitalize
- # Calculating valid languages took quite long, so we calculate it once
- # in initialization instead of each time it is used.
- self._validlanguages = []
- for language in self.languages():
- if not language[:1].upper() + language[1:] in self.namespaces():
- self._validlanguages.append(language)
-
- def logged_in(self, sysop=False):
- """Return True if logged in with specified privileges, otherwise False.
-
- @param sysop: if True, require sysop privileges.
-
- """
- if self.userinfo['name'] != self._username[sysop]:
- return False
- return (not sysop) or 'sysop' in self.userinfo['groups']
-
- def loggedInAs(self, sysop = False):
- """Return the current username if logged in, otherwise return None.
-
- DEPRECATED (use .user() method instead)
-
- """
- logger.debug("Site.loggedInAs() method is deprecated.")
- return self.logged_in(sysop) and self.user()
-
- def login(self, sysop=False):
- """Log the user in if not already logged in."""
- if not hasattr(self, "_siteinfo"):
- self._getsiteinfo()
- # check whether a login cookie already exists for this user
- if hasattr(self, "_userinfo"):
- if self.userinfo['name'] == self._username[sysop]:
- return
- if not self.logged_in(sysop):
- loginMan = api.LoginManager(site=self, sysop=sysop,
- user=self._username[sysop])
- if loginMan.login(retry = True):
- self._username[sysop] = loginMan.username
- if hasattr(self, "_userinfo"):
- del self._userinfo
- self.getuserinfo()
-
- forceLogin = login # alias for backward-compatibility
-
- def getuserinfo(self):
- """Retrieve userinfo from site and store in _userinfo attribute.
-
- self._userinfo will be a dict with the following keys and values:
-
- - id: user id (numeric str)
- - name: username (if user is logged in)
- - anon: present if user is not logged in
- - groups: list of groups (could be empty)
- - rights: list of rights (could be empty)
- - message: present if user has a new message on talk page
- - blockinfo: present if user is blocked (dict)
-
- """
- if (not hasattr(self, "_userinfo")
- or "rights" not in self._userinfo
- or self._userinfo['name']
- != self._username["sysop" in self._userinfo["groups"]]):
- uirequest = api.Request(
- site=self,
- action="query",
- meta="userinfo",
- uiprop="blockinfo|hasmsg|groups|rights"
- )
- uidata = uirequest.submit()
- assert 'query' in uidata, \
- "API userinfo response lacks 'query' key"
- assert 'userinfo' in uidata['query'], \
- "API userinfo response lacks 'userinfo' key"
- self._userinfo = uidata['query']['userinfo']
- return self._userinfo
-
- userinfo = property(fget=getuserinfo, doc=getuserinfo.__doc__)
-
- def is_blocked(self, sysop=False):
- """Return true if and only if user is blocked.
-
- @param sysop: If true, log in to sysop account (if available)
-
- """
- if not self.logged_in(sysop):
- self.login(sysop)
- return 'blockinfo' in self._userinfo
-
- def isBlocked(self, sysop=False):
- """Deprecated synonym for is_blocked"""
- logger.debug(
- "Site method 'isBlocked' should be changed to 'is_blocked'")
- return self.is_blocked(sysop)
-
- def checkBlocks(self, sysop = False):
- """Check if the user is blocked, and raise an exception if so."""
- if self.is_blocked(sysop):
- # User blocked
- raise UserBlocked('User is blocked in site %s' % self)
-
- def has_right(self, right, sysop=False):
- """Return true if and only if the user has a specific right.
-
- Possible values of 'right' may vary depending on wiki settings,
- but will usually include:
-
- * Actions: edit, move, delete, protect, upload
- * User levels: autoconfirmed, sysop, bot
-
- """
- if not self.logged_in(sysop):
- self.login(sysop)
- return right.lower() in self._userinfo['rights']
-
- def isAllowed(self, right, sysop=False):
- """Deprecated; retained for backwards-compatibility"""
- logger.debug("Site.isAllowed() method is deprecated; use has_right()")
- return self.has_right(right, sysop)
-
- def has_group(self, group, sysop=False):
- """Return true if and only if the user is a member of specified group.
-
- Possible values of 'group' may vary depending on wiki settings,
- but will usually include bot.
-
- """
- if not self.logged_in(sysop):
- self.login(sysop)
- return group.lower() in self._userinfo['groups']
-
- def messages(self, sysop=False):
- """Returns true if the user has new messages, and false otherwise."""
- if not self.logged_in(sysop):
- self.login(sysop)
- return 'hasmsg' in self._userinfo
-
- def mediawiki_message(self, key):
- """Return the MediaWiki message text for key "key" """
- if not key in self._msgcache:
- msg_query = api.QueryGenerator(site=self, meta="allmessages",
- amfilter=key)
- for msg in msg_query:
- if msg['name'] == key and not 'missing' in msg:
- self._msgcache[key] = msg['*']
- break
- else:
- raise KeyError("Site %(self)s has no message '%(key)s'"
- % locals())
- return self._msgcache[key]
-
- def has_mediawiki_message(self, key):
- """Return True iff this site defines a MediaWiki message for 'key'."""
- try:
- v = self.mediawiki_message(key)
- return True
- except KeyError:
- return False
-
- def getcurrenttimestamp(self):
- """Return (Mediawiki) timestamp, {{CURRENTTIMESTAMP}}, the server time.
-
- Format is yyyymmddhhmmss
-
- """
- r = api.Request(site=self,
- action="parse",
- text="{{CURRENTTIMESTAMP}}")
- result = r.submit()
- return re.search('\d+', result['parse']['text']['*']).group()
-
- def _getsiteinfo(self):
- """Retrieve siteinfo and namespaces from site."""
- sirequest = api.Request(
- site=self,
- action="query",
- meta="siteinfo",
- siprop="general|namespaces|namespacealiases"
- )
- try:
- sidata = sirequest.submit()
- except api.APIError:
- # hack for older sites that don't support 1.12 properties
- # probably should delete if we're not going to support pre-1.12
- sirequest = api.Request(
- site=self,
- action="query",
- meta="siteinfo",
- siprop="general|namespaces"
- )
- sidata = sirequest.submit()
-
- assert 'query' in sidata, \
- "API siteinfo response lacks 'query' key"
- sidata = sidata['query']
- assert 'general' in sidata, \
- "API siteinfo response lacks 'general' key"
- assert 'namespaces' in sidata, \
- "API siteinfo response lacks 'namespaces' key"
- self._siteinfo = sidata['general']
- nsdata = sidata['namespaces']
- for nskey in nsdata:
- ns = int(nskey)
- if ns in self._namespaces:
- if nsdata[nskey]["*"] in self._namespaces[ns]:
- continue
- # this is the preferred form so it goes at front of list
- self._namespaces[ns].insert(0, nsdata[nskey]["*"])
- else:
- self._namespaces[ns] = [nsdata[nskey]["*"]]
- if 'namespacealiases' in sidata:
- aliasdata = sidata['namespacealiases']
- for item in aliasdata:
- if item["*"] in self._namespaces[int(item['id'])]:
- continue
- # this is a less preferred form so it goes at the end
- self._namespaces[int(item['id'])].append(item["*"])
-
- @property
- def siteinfo(self):
- """Site information dict."""
-
- if not hasattr(self, "_siteinfo"):
- self._getsiteinfo()
- return self._siteinfo
-
- def case(self):
- """Return this site's capitalization rule."""
-
- return self.siteinfo['case']
-
- def language(self):
- """Return the code for the language of this Site."""
-
- return self.siteinfo['lang']
-
- lang = property(fget=language, doc=language.__doc__)
-
- def namespaces(self):
- """Return dict of valid namespaces on this wiki."""
-
- if not hasattr(self, "_siteinfo"):
- self._getsiteinfo()
- return self._namespaces
-
- def namespace(self, num, all=False):
- """Return string containing local name of namespace 'num'.
-
- If optional argument 'all' is true, return a list of all recognized
- values for this namespace.
-
- """
- if all:
- return self.namespaces()[num]
- return self.namespaces()[num][0]
-
- def live_version(self):
- """Return the 'real' version number found on [[Special:Version]]
-
- Return value is a tuple (int, int, str) of the major and minor
- version numbers and any other text contained in the version.
-
- """
- versionstring = self.siteinfo['generator']
- m = re.match(r"^MediaWiki ([0-9]+)\.([0-9]+)(.*)$", versionstring)
- if m:
- return (int(m.group(1)), int(m.group(2)), m.group(3))
- else:
- return None
-
- def loadpageinfo(self, page):
- """Load page info from api and save in page attributes"""
- title = page.title(withSection=False)
- query = api.PropertyGenerator("info", site=self,
- titles=title.encode(self.encoding()),
- inprop="protection")
- for pageitem in query:
- if pageitem['title'] != title:
- raise Error(
- u"loadpageinfo: Query on %s returned data on '%s'"
- % (page, pageitem['title']))
- api.update_page(page, pageitem)
-
- def loadimageinfo(self, page, history=False):
- """Load image info from api and save in page attributes
-
- @param history: if true, return the image's version history
-
- """
- title = page.title(withSection=False)
- query = api.PropertyGenerator("imageinfo", site=self,
- titles=title.encode(self.encoding()),
- iiprop=["timestamp", "user", "comment",
- "url", "size", "sha1", "mime",
- "metadata", "archivename"])
- if history:
- query.request["iilimit"] = "max"
- for pageitem in query:
- if pageitem['title'] != title:
- raise Error(
- u"loadpageinfo: Query on %s returned data on '%s'"
- % (page, pageitem['title']))
- api.update_page(page, pageitem)
- if history:
- return pageitem['imageinfo']
-
- def page_exists(self, page):
- """Return True if and only if page is an existing page on site."""
- if not hasattr(page, "_pageid"):
- self.loadpageinfo(page)
- return page._pageid > 0
-
- def page_restrictions(self, page):
- """Returns a dictionary reflecting page protections"""
- if not self.page_exists(page):
- raise NoPage(u'No page %s.' % page)
- if not hasattr(page, "_protection"):
- self.loadpageinfo(page)
- return page._protection
-
- def page_can_be_edited(self, page):
- """
- Returns True if and only if:
- - page is unprotected, and bot has an account for this site, or
- - page is protected, and bot has a sysop account for this site.
-
- """
- rest = self.page_restrictions(page)
- sysop_protected = rest.has_key('edit') and rest['edit'][0] == 'sysop'
- try:
- api.LoginManager(site=self, sysop=sysop_protected)
- except NoUsername:
- return False
- return True
-
- def page_isredirect(self, page):
- """Return True if and only if page is a redirect."""
- if not hasattr(page, "_redir"):
- self.loadpageinfo(page)
- return bool(page._redir)
-
- def getredirtarget(self, page):
- """Return Page object for the redirect target of page."""
- if not hasattr(page, "_redir"):
- self.loadpageinfo(page)
- if not page._redir:
- raise pywikibot.IsNotRedirectPage(page.title())
- title = page.title(withSection=False)
- query = api.Request(site=self, action="query", property="info",
- inprop="protection|talkid|subjectid",
- titles=title.encode(self.encoding()),
- redirects="")
- result = query.submit()
- if "query" not in result or "redirects" not in result["query"]:
- raise RuntimeError(
- "getredirtarget: No 'redirects' found for page %s."
- % title)
- redirmap = dict((item['from'], item['to'])
- for item in result['query']['redirects'])
- if title not in redirmap:
- raise RuntimeError(
- "getredirtarget: 'redirects' contains no key for page %s."
- % title)
- if "pages" not in result['query']:
- # no "pages" element indicates a circular redirect
- raise pywikibot.CircularRedirect(redirmap[title])
- for pagedata in result['query']['pages'].values():
- # there should be only one value in 'pages', and it is the target
- if pagedata['title'] not in redirmap.values():
- raise RuntimeError(
- "getredirtarget: target page '%s' not found in 'redirects'"
- % pagedata['title'])
- target = pywikibot.Page(self, pagedata['title'], pagedata['ns'])
- api.update_page(target, pagedata)
- page._redir = target
-
- def preloadpages(self, pagelist, groupsize=60):
- """Return a generator to a list of preloaded pages.
-
- Note that [at least in current implementation] pages may be iterated
- in a different order than in the underlying pagelist.
-
- @param pagelist: an iterable that returns Page objects
- @param groupsize: how many Pages to query at a time
- @type groupsize: int
-
- """
- from pywikibot.tools import itergroup
- for sublist in itergroup(pagelist, groupsize):
- pageids = [str(p._pageid) for p in sublist
- if hasattr(p, "_pageid")
- and p._pageid > 0]
- cache = dict((p.title(withSection=False), p) for p in sublist)
- rvgen = api.PropertyGenerator("revisions|info", site=self)
- rvgen.limit = -1
- if len(pageids) == len(sublist):
- # only use pageids if all pages have them
- rvgen.request["pageids"] = "|".join(pageids)
- else:
- rvgen.request["titles"] = "|".join(cache.keys())
- rvgen.request[u"rvprop"] = \
- u"ids|flags|timestamp|user|comment|content"
- logger.info(u"Retrieving %s pages from %s."
- % (len(cache), self)
- )
- for pagedata in rvgen:
- logger.debug("Preloading %s" % pagedata)
- try:
- if pagedata['title'] not in cache:
- raise Error(
- u"preloadpages: Query returned unexpected title '%s'"
- % pagedata['title']
- )
- except KeyError:
- logger.debug("No 'title' in %s" % pagedata)
- logger.debug("pageids=%s" % pageids)
- logger.debug("titles=%s" % cache.keys())
- continue
- page = cache[pagedata['title']]
- api.update_page(page, pagedata)
- yield page
-
- def token(self, page, tokentype):
- """Return token retrieved from wiki to allow changing page content.
-
- @param page: the Page for which a token should be retrieved
- @param tokentype: the type of token (e.g., "edit", "move", "delete");
- see API documentation for full list of types
-
- """
- query = api.PropertyGenerator("info|revisions", site=self,
- titles=page.title(withSection=False),
- intoken=tokentype)
- for item in query:
- if item['title'] != page.title(withSection=False):
- raise Error(
- u"token: Query on page %s returned data on page [[%s]]"
- % (page.title(withSection=False, asLink=True),
- item['title']))
- api.update_page(page, item)
- logging.debug(str(item))
- return item[tokentype + "token"]
-
- # following group of methods map more-or-less directly to API queries
-
- def pagebacklinks(self, page, followRedirects=False, filterRedirects=None,
- namespaces=None):
- """Iterate all pages that link to the given page.
-
- @param page: The Page to get links to.
- @param followRedirects: Also return links to redirects pointing to
- the given page.
- @param filterRedirects: If True, only return redirects to the given
- page. If False, only return non-redirect links. If None, return
- both (no filtering).
- @param namespaces: If present, only return links from the namespaces
- in this list.
-
- """
- bltitle = page.title(withSection=False).encode(self.encoding())
- blgen = api.PageGenerator("backlinks", gbltitle=bltitle, site=self)
- if isinstance(namespaces, list):
- blgen.request["gblnamespace"] = u"|".join(unicode(ns)
- for ns in namespaces)
- elif namespaces is not None:
- blgen.request["gblnamespace"] = str(namespaces)
- if filterRedirects is not None:
- blgen.request["gblfilterredir"] = filterRedirects and "redirects"\
- or "nonredirects"
- if followRedirects:
- # bug: see http://bugzilla.wikimedia.org/show_bug.cgi?id=16218
- # links identified by MediaWiki as redirects may not really be,
- # so we have to check each "redirect" page and see if it
- # really redirects to this page
- blgen.request["gblfilterredir"] = "nonredirects"
- redirgen = api.PageGenerator("backlinks", gbltitle=bltitle,
- site=self, gblfilterredir="redirects")
- if "gblnamespace" in blgen.request:
- redirgen.request["gblnamespace"] = blgen.request["gblnamespace"]
- genlist = [blgen]
- for redir in redirgen:
- if redir.getRedirectTarget() == page:
- genlist.append(
- self.pagebacklinks(
- redir, True, None, namespaces))
- import itertools
- return itertools.chain(*genlist)
- return blgen
-
- def page_embeddedin(self, page, filterRedirects=None, namespaces=None):
- """Iterate all pages that embedded the given page as a template.
-
- @param page: The Page to get inclusions for.
- @param filterRedirects: If True, only return redirects that embed
- the given page. If False, only return non-redirect links. If
- None, return both (no filtering).
- @param namespaces: If present, only return links from the namespaces
- in this list.
-
- """
- eititle = page.title(withSection=False).encode(self.encoding())
- eigen = api.PageGenerator("embeddedin", geititle=eititle, site=self)
- if isinstance(namespaces, list):
- eigen.request["geinamespace"] = u"|".join(unicode(ns)
- for ns in namespaces)
- elif namespaces is not None:
- eigen.request["geinamespace"] = str(namespaces)
- if filterRedirects is not None:
- eigen.request["geifilterredir"] = filterRedirects and "redirects"\
- or "nonredirects"
- return eigen
-
- def pagereferences(self, page, followRedirects=False, filterRedirects=None,
- withTemplateInclusion=True, onlyTemplateInclusion=False,
- namespaces=None):
- """Convenience method combining pagebacklinks and page_embeddedin."""
-
- if onlyTemplateInclusion:
- return self.page_embeddedin(page, namespaces=namespaces)
- if not withTemplateInclusion:
- return self.pagebacklinks(page, followRedirects,
- namespaces=namespaces)
- import itertools
- return itertools.chain(
- self.pagebacklinks(page, followRedirects,
- filterRedirects, namespaces=namespaces),
- self.page_embeddedin(page, filterRedirects,
- namespaces=namespaces)
- )
-
- def pagelinks(self, page, namespaces=None, follow_redirects=False,
- limit=None):
- """Iterate internal wikilinks contained (or transcluded) on page.
-
- @param namespaces: Only iterate pages in these namespaces (default: all)
- @type namespaces: list of ints
- @param follow_redirects: if True, yields the target of any redirects,
- rather than the redirect page
-
- """
- plgen = api.PageGenerator("links", site=self)
- if isinstance(limit, int):
- plgen.limit = limit
- if hasattr(page, "_pageid"):
- plgen.request['pageids'] = str(page._pageid)
- else:
- pltitle = page.title(withSection=False).encode(self.encoding())
- plgen.request['titles'] = pltitle
- if follow_redirects:
- plgen.request['redirects'] = ''
- if isinstance(namespaces, list):
- plgen.request["gplnamespace"] = u"|".join(unicode(ns)
- for ns in namespaces)
- elif namespaces is not None:
- plgen.request["gplnamespace"] = str(namespaces)
- return plgen
-
- @deprecate_arg("withSortKey", None) # Sortkey doesn't work with generator
- def pagecategories(self, page, withSortKey=None):
- """Iterate categories to which page belongs."""
-
- clgen = api.CategoryPageGenerator("categories", site=self)
- if hasattr(page, "_pageid"):
- clgen.request['pageids'] = str(page._pageid)
- else:
- cltitle = page.title(withSection=False).encode(self.encoding())
- clgen.request['titles'] = cltitle
- return clgen
-
- def pageimages(self, page):
- """Iterate images used (not just linked) on the page."""
-
- imtitle = page.title(withSection=False).encode(self.encoding())
- imgen = api.ImagePageGenerator("images", titles=imtitle, site=self)
- return imgen
-
- def pagetemplates(self, page, namespaces=None):
- """Iterate templates transcluded (not just linked) on the page."""
-
- tltitle = page.title(withSection=False).encode(self.encoding())
- tlgen = api.PageGenerator("templates", titles=tltitle, site=self)
- if isinstance(namespaces, list):
- tlgen.request["gtlnamespace"] = u"|".join(unicode(ns)
- for ns in namespaces)
- elif namespaces is not None:
- tlgen.request["gtlnamespace"] = str(namespaces)
- return tlgen
-
- def categorymembers(self, category, namespaces=None, limit=None):
- """Iterate members of specified category.
-
- @param category: The Category to iterate.
- @param namespaces: If present, only return category members from
- these namespaces. For example, use namespaces=[14] to yield
- subcategories, use namespaces=[6] to yield image files, etc. Note,
- however, that the iterated values are always Page objects, even
- if in the Category or Image namespace.
- @type namespaces: list of ints
- @param limit: maximum number of pages to iterate (default: all)
- @type limit: int
-
- """
- if category.namespace() != 14:
- raise Error(
- u"categorymembers: non-Category page '%s' specified"
- % category.title())
- cmtitle = category.title(withSection=False).encode(self.encoding())
- cmgen = api.PageGenerator("categorymembers", gcmtitle=cmtitle,
- gcmprop="ids|title|sortkey", site=self)
- if isinstance(namespaces, list):
- cmgen.request["gcmnamespace"] = u"|".join(unicode(ns)
- for ns in namespaces)
- elif namespaces is not None:
- cmgen.request["gcmnamespace"] = str(namespaces)
- if isinstance(limit, int):
- cmgen.limit = limit
- return cmgen
-
- def loadrevisions(self, page=None, getText=False, revids=None,
- limit=None, startid=None, endid=None, starttime=None,
- endtime=None, rvdir=None, user=None, excludeuser=None,
- section=None, sysop=False):
- """Retrieve and store revision information.
-
- By default, retrieves the last (current) revision of the page,
- I{unless} any of the optional parameters revids, startid, endid,
- starttime, endtime, rvdir, user, excludeuser, or limit are
- specified. Unless noted below, all parameters not specified
- default to False.
-
- If rvdir is False or not specified, startid must be greater than
- endid if both are specified; likewise, starttime must be greater
- than endtime. If rvdir is True, these relationships are reversed.
-
- @param page: retrieve revisions of this Page (required unless ids
- is specified)
- @param getText: if True, retrieve the wiki-text of each revision;
- otherwise, only retrieve the revision metadata (default)
- @param section: if specified, retrieve only this section of the text
- (getText must be True); section must be given by number (top of
- the article is section 0), not name
- @type section: int
- @param revids: retrieve only the specified revision ids (required
- unless page is specified)
- @type revids: list of ints
- @param limit: Retrieve no more than this number of revisions
- @type limit: int
- @param startid: retrieve revisions starting with this revid
- @param endid: stop upon retrieving this revid
- @param starttime: retrieve revisions starting at this timestamp
- @param endtime: stop upon reaching this timestamp
- @param rvdir: if false, retrieve newest revisions first (default);
- if true, retrieve earliest first
- @param user: retrieve only revisions authored by this user
- @param excludeuser: retrieve all revisions not authored by this user
- @param sysop: if True, switch to sysop account (if available) to
- retrieve this page
-
- """
- latest = (revids is None and
- startid is None and
- endid is None and
- starttime is None and
- endtime is None and
- rvdir is None and
- user is None and
- excludeuser is None and
- limit is None) # if True, we are retrieving current revision
-
- # check for invalid argument combinations
- if page is None and revids is None:
- raise ValueError(
- "loadrevisions: either page or revids argument required")
- if (startid is not None or endid is not None) and \
- (starttime is not None or endtime is not None):
- raise ValueError(
- "loadrevisions: startid/endid combined with starttime/endtime")
- if starttime is not None and endtime is not None:
- if rvdir and starttime >= endtime:
- raise ValueError(
- "loadrevisions: starttime > endtime with rvdir=True")
- if (not rvdir) and endtime >= starttime:
- raise ValueError(
- "loadrevisions: endtime > starttime with rvdir=False")
- if startid is not None and endid is not None:
- if rvdir and startid >= endid:
- raise ValueError(
- "loadrevisions: startid > endid with rvdir=True")
- if (not rvdir) and endid >= startid:
- raise ValueError(
- "loadrevisions: endid > startid with rvdir=False")
-
- # assemble API request
- if revids is None:
- rvtitle = page.title(withSection=False).encode(self.encoding())
- rvgen = api.PropertyGenerator(u"info|revisions", titles=rvtitle,
- site=self)
- else:
- if isinstance(revids, (int, basestring)):
- ids = unicode(revids)
- else:
- ids = u"|".join(unicode(r) for r in revids)
- rvgen = api.PropertyGenerator(u"info|revisions", revids=ids,
- site=self)
- if getText:
- rvgen.request[u"rvprop"] = \
- u"ids|flags|timestamp|user|comment|content"
- if section is not None:
- rvgen.request[u"rvsection"] = unicode(section)
- if latest or "revids" in rvgen.request:
- rvgen.limit = -1 # suppress use of rvlimit parameter
- elif isinstance(limit, int):
- rvgen.limit = limit
- if rvdir:
- rvgen.request[u"rvdir"] = u"newer"
- elif rvdir is not None:
- rvgen.request[u"rvdir"] = u"older"
- if startid:
- rvgen.request[u"rvstartid"] = startid
- if endid:
- rvgen.request[u"rvendid"] = endid
- if starttime:
- rvgen.request[u"rvstart"] = starttime
- if endtime:
- rvgen.request[u"rvend"] = endtime
- if user:
- rvgen.request[u"rvuser"] = user
- elif excludeuser:
- rvgen.request[u"rvexcludeuser"] = excludeuser
- # TODO if sysop: something
- rvgen.continuekey = "revisions"
- for pagedata in rvgen:
- if page is not None:
- if pagedata['title'] != page.title(withSection=False):
- raise Error(
- u"loadrevisions: Query on %s returned data on '%s'"
- % (page, pagedata['title']))
- if pagedata.has_key('missing'):
- raise NoPage(u'Page %s does not exist'
- % page.title(asLink=True))
- else:
- page = Page(self, pagedata['title'])
- api.update_page(page, pagedata)
-
- def pageinterwiki(self, page):
- # No such function in the API (this method isn't called anywhere)
- raise NotImplementedError
-
- def pagelanglinks(self, page):
- """Iterate all interlanguage links on page, yielding Link objects."""
- lltitle = page.title(withSection=False)
- llquery = api.PropertyGenerator("langlinks",
- titles=lltitle.encode(self.encoding()),
- site=self)
- for pageitem in llquery:
- if pageitem['title'] != lltitle:
- raise Error(
- u"getlanglinks: Query on %s returned data on '%s'"
- % (page, pageitem['title']))
- if 'langlinks' not in pageitem:
- continue
- for linkdata in pageitem['langlinks']:
- yield pywikibot.Link(linkdata['*'],
- source=pywikibot.Site(linkdata['lang']))
-
- def page_extlinks(self, page):
- """Iterate all external links on page, yielding URL strings."""
- eltitle = page.title(withSection=False)
- elquery = api.PropertyGenerator("extlinks",
- titles=eltitle.encode(self.encoding()),
- site=self)
- for pageitem in elquery:
- if pageitem['title'] != eltitle:
- raise RuntimeError(
- "getlanglinks: Query on %s returned data on '%s'"
- % (page, pageitem['title']))
- if 'extlinks' not in pageitem:
- continue
- for linkdata in pageitem['extlinks']:
- yield linkdata['*']
-
- @deprecate_arg("throttle", None)
- @deprecate_arg("includeredirects", "filterredir")
- def allpages(self, start="!", prefix="", namespace=0, filterredir=None,
- filterlanglinks=None, minsize=None, maxsize=None,
- protect_type=None, protect_level=None, limit=None,
- reverse=False, includeredirects=None):
- """Iterate pages in a single namespace.
-
- Note: parameters includeRedirects and throttle are deprecated and
- included only for backwards compatibility.
-
- @param start: Start at this title (page need not exist).
- @param prefix: Only yield pages starting with this string.
- @param namespace: Iterate pages from this (single) namespace
- (default: 0)
- @param filterredir: if True, only yield redirects; if False (and not
- None), only yield non-redirects (default: yield both)
- @param filterlanglinks: if True, only yield pages with language links;
- if False (and not None), only yield pages without language links
- (default: yield both)
- @param minsize: if present, only yield pages at least this many
- bytes in size
- @param maxsize: if present, only yield pages at most this many bytes
- in size
- @param protect_type: only yield pages that have a protection of the
- specified type
- @type protect_type: str
- @param protect_level: only yield pages that have protection at this
- level; can only be used if protect_type is specified
- @param limit: maximum number of pages to iterate (default: iterate
- all pages in namespace)
- @param reverse: if True, iterate in reverse Unicode lexigraphic
- order (default: iterate in forward order)
- @param includeredirects: DEPRECATED, use filterredirs instead
-
- """
- if not isinstance(namespace, int):
- raise Error("allpages: only one namespace permitted.")
- if includeredirects is not None:
- logger.debug(
-"allpages: 'includeRedirects' argument is deprecated; use 'filterredirs'.")
- if includeredirects:
- if includeredirects == "only":
- filterredirs = True
- else:
- filterredirs = None
- else:
- filterredirs = False
-
- apgen = api.PageGenerator("allpages", gapnamespace=str(namespace),
- gapfrom=start, site=self)
- if prefix:
- apgen.request["gapprefix"] = prefix
- if filterredir is not None:
- apgen.request["gapfilterredir"] = (filterredir
- and "redirects"
- or "nonredirects")
- if filterlanglinks is not None:
- apgen.request["gapfilterlanglinks"] = (filterlanglinks
- and "withlanglinks"
- or "withoutlanglinks")
- if isinstance(minsize, int):
- apgen.request["gapminsize"] = str(minsize)
- if isinstance(maxsize, int):
- apgen.request["gapmaxsize"] = str(maxsize)
- if isinstance(protect_type, basestring):
- apgen.request["gapprtype"] = protect_type
- if isinstance(protect_level, basestring):
- apgen.request["gapprlevel"] = protect_level
- if isinstance(limit, int):
- apgen.limit = limit
- if reverse:
- apgen.request["gapdir"] = "descending"
- return apgen
-
- def prefixindex(self, prefix, namespace=0, includeredirects=True):
- """Yield all pages with a given prefix. Deprecated.
-
- Use allpages() with the prefix= parameter instead of this method.
-
- """
- logger.debug("Site.prefixindex() is deprecated; use allpages instead.")
- return self.allpages(prefix=prefix, namespace=namespace,
- includeredirects=includeredirects)
-
-
- def alllinks(self, start="!", prefix="", namespace=0, unique=False,
- limit=None, fromids=False):
- """Iterate all links to pages (which need not exist) in one namespace.
-
- Note that, in practice, links that were found on pages that have
- been deleted may not have been removed from the links table, so this
- method can return false positives.
-
- @param start: Start at this title (page need not exist).
- @param prefix: Only yield pages starting with this string.
- @param namespace: Iterate pages from this (single) namespace
- (default: 0)
- @param unique: If True, only iterate each link title once (default:
- iterate once for each linking page)
- @param limit: maximum number of pages to iterate (default: iterate
- all pages in namespace)
- @param fromids: if True, include the pageid of the page containing
- each link (default: False) as the '_fromid' attribute of the Page;
- cannot be combined with unique
-
- """
- if unique and fromids:
- raise Error("alllinks: unique and fromids cannot both be True.")
- if not isinstance(namespace, int):
- raise Error("alllinks: only one namespace permitted.")
- algen = api.ListGenerator("alllinks", alnamespace=str(namespace),
- alfrom=start, site=self)
- if prefix:
- algen.request["alprefix"] = prefix
- if isinstance(limit, int):
- algen.limit = limit
- if unique:
- algen.request["alunique"] = ""
- if fromids:
- algen.request["alprop"] = "title|ids"
- for link in algen:
- p = pywikibot.Page(self, link['title'], link['ns'])
- if fromids:
- p._fromid = link['fromid']
- yield p
-
- def allcategories(self, start="!", prefix="", limit=None,
- reverse=False):
- """Iterate categories used (which need not have a Category page).
-
- Iterator yields Category objects. Note that, in practice, links that
- were found on pages that have been deleted may not have been removed
- from the database table, so this method can return false positives.
-
- @param start: Start at this category title (category need not exist).
- @param prefix: Only yield categories starting with this string.
- @param limit: maximum number of categories to iterate (default:
- iterate all)
- @param reverse: if True, iterate in reverse Unicode lexigraphic
- order (default: iterate in forward order)
-
- """
- acgen = api.CategoryPageGenerator("allcategories",
- gacfrom=start, site=self)
- if prefix:
- acgen.request["gacprefix"] = prefix
- if isinstance(limit, int):
- acgen.limit = limit
- if reverse:
- acgen.request["gacdir"] = "descending"
- return acgen
-
- def categories(self, number=10, repeat=False):
- """Deprecated; retained for backwards-compatibility"""
- logger.debug(
- "Site.categories() method is deprecated; use .allcategories()")
- if repeat:
- limit = None
- else:
- limit = number
- return self.allcategories(limit=limit)
-
- def allusers(self, start="!", prefix="", limit=None, group=None):
- """Iterate registered users, ordered by username.
-
- Iterated values are dicts containing 'name', 'editcount',
- 'registration', and (sometimes) 'groups' keys. 'groups' will be
- present only if the user is a member of at least 1 group, and will
- be a list of unicodes; all the other values are unicodes and should
- always be present.
-
- @param start: start at this username (name need not exist)
- @param prefix: only iterate usernames starting with this substring
- @param limit: maximum number of users to iterate (default: all)
- @param group: only iterate users that are members of this group
- @type group: str
-
- """
- augen = api.ListGenerator("allusers", aufrom=start,
- auprop="editcount|groups|registration",
- site=self)
- if prefix:
- augen.request["auprefix"] = prefix
- if group:
- augen.request["augroup"] = group
- if isinstance(limit, int):
- augen.limit = limit
- return augen
-
- def allimages(self, start="!", prefix="", minsize=None, maxsize=None,
- limit=None, reverse=False, sha1=None, sha1base36=None):
- """Iterate all images, ordered by image title.
-
- Yields ImagePages, but these pages need not exist on the wiki.
-
- @param start: start at this title (name need not exist)
- @param prefix: only iterate titles starting with this substring
- @param limit: maximum number of titles to iterate (default: all)
- @param minsize: only iterate images of at least this many bytes
- @param maxsize: only iterate images of no more than this many bytes
- @param reverse: if True, iterate in reverse lexigraphic order
- @param sha1: only iterate image (it is theoretically possible there
- could be more than one) with this sha1 hash
- @param sha1base36: same as sha1 but in base 36
-
- """
- aigen = api.ImagePageGenerator("allimages", gaifrom=start,
- site=self)
- if prefix:
- aigen.request["gaiprefix"] = prefix
- if isinstance(limit, int):
- aigen.limit = limit
- if isinstance(minsize, int):
- aigen.request["gaiminsize"] = str(minsize)
- if isinstance(maxsize, int):
- aigen.request["gaimaxsize"] = str(maxsize)
- if reverse:
- aigen.request["gaidir"] = "descending"
- if sha1:
- aigen.request["gaisha1"] = sha1
- if sha1base36:
- aigen.request["gaisha1base36"] = sha1base36
- return aigen
-
- def blocks(self, starttime=None, endtime=None, reverse=False,
- blockids=None, users=None, limit=None):
- """Iterate all current blocks, in order of creation.
-
- Note that logevents only logs user blocks, while this method
- iterates all blocks including IP ranges. The iterator yields dicts
- containing keys corresponding to the block properties (see
- http://www.mediawiki.org/wiki/API:Query_-_Lists for documentation).
-
- @param starttime: start iterating at this timestamp
- @param endtime: stop iterating at this timestamp
- @param reverse: if True, iterate oldest blocks first (default: newest)
- @param blockids: only iterate blocks with these id numbers
- @param users: only iterate blocks affecting these usernames or IPs
- @param limit: maximum number of blocks to iterate (default: all)
-
- """
- if starttime and endtime:
- if reverse:
- if starttime > endtime:
- raise pywikibot.Error(
- "blocks: starttime must be before endtime with reverse=True")
- else:
- if endtime > starttime:
- raise pywikibot.Error(
- "blocks: endtime must be before starttime with reverse=False")
- bkgen = api.ListGenerator("blocks", site=self)
- bkgen.request["bkprop"] = \
- "id|user|by|timestamp|expiry|reason|range|flags"
- if starttime:
- bkgen.request["bkstart"] = starttime
- if endtime:
- bkgen.request["bkend"] = endtime
- if reverse:
- bkgen.request["bkdir"] = "newer"
- if blockids:
- bkgen.request["bkids"] = blockids
- if users:
- bkgen.request["bkusers"] = users
- if isinstance(limit, int):
- bkgen.limit = limit
- return bkgen
-
- def exturlusage(self, url, protocol="http", namespaces=None,
- limit=None):
- """Iterate Pages that contain links to the given URL.
-
- @param url: The URL to search for (without the protocol prefix);
- this many include a '*' as a wildcard, only at the start of the
- hostname
- @param protocol: The protocol prefix (default: "http")
- @param namespaces: Only iterate pages in these namespaces (default: all)
- @type namespaces: list of ints
- @param limit: Only iterate this many linking pages (default: all)
-
- """
- eugen = api.PageGenerator("exturlusage", geuquery=url,
- geuprotocol=protocol, site=self)
- if isinstance(namespaces, list):
- eugen.request["geunamespace"] = u"|".join(unicode(ns)
- for ns in namespaces)
- elif namespaces is not None:
- eugen.request["geunamespace"] = str(namespaces)
- if isinstance(limit, int):
- eugen.limit = limit
- return eugen
-
- def imageusage(self, image, namespaces=None, filterredir=None,
- limit=None):
- """Iterate Pages that contain links to the given ImagePage.
-
- @param image: the image to search for (ImagePage need not exist on the wiki)
- @type image: ImagePage
- @param namespaces: Only iterate pages in these namespaces (default: all)
- @type namespaces: list of ints
- @param filterredir: if True, only yield redirects; if False (and not
- None), only yield non-redirects (default: yield both)
- @param limit: Only iterate this many linking pages (default: all)
-
- """
- iugen = api.PageGenerator("imageusage", site=self,
- giutitle=image.title(withSection=False))
- if isinstance(namespaces, list):
- iugen.request["giunamespace"] = u"|".join(unicode(ns)
- for ns in namespaces)
- elif namespaces is not None:
- iugen.request["giunamespace"] = str(namespaces)
- if isinstance(limit, int):
- iugen.limit = limit
- if filterredir is not None:
- iugen.request["giufilterredir"] = (filterredir and "redirects"
- or "nonredirects")
- return iugen
-
- def logevents(self, logtype=None, user=None, page=None,
- start=None, end=None, reverse=False, limit=None):
- """Iterate all log entries.
-
- @param logtype: only iterate entries of this type (see wiki
- documentation for available types, which will include "block",
- "protect", "rights", "delete", "upload", "move", "import",
- "patrol", "merge")
- @param user: only iterate entries that match this user name
- @param page: only iterate entries affecting this page
- @param start: only iterate entries from and after this timestamp
- @param end: only iterate entries up to and through this timestamp
- @param reverse: if True, iterate oldest entries first (default: newest)
- @param limit: only iterate up to this many entries
-
- """
- if start and end:
- if reverse:
- if end < start:
- raise Error(
- "logevents: end must be later than start with reverse=True")
- else:
- if start < end:
- raise Error(
- "logevents: start must be later than end with reverse=False")
- legen = api.ListGenerator("logevents", site=self)
- if logtype is not None:
- legen.request["letype"] = logtype
- if user is not None:
- legen.request["leuser"] = user
- if page is not None:
- legen.request["letitle"] = page.title(withSection=False)
- if start is not None:
- legen.request["lestart"] = start
- if end is not None:
- legen.request["leend"] = end
- if reverse:
- legen.request["ledir"] = "newer"
- if isinstance(limit, int):
- legen.limit = limit
- return legen
-
- def recentchanges(self, start=None, end=None, reverse=False, limit=None,
- namespaces=None, pagelist=None, changetype=None,
- showMinor=None, showBot=None, showAnon=None,
- showRedirects=None, showPatrolled=None):
- """Iterate recent changes.
-
- @param start: timestamp to start listing from
- @param end: timestamp to end listing at
- @param reverse: if True, start with oldest changes (default: newest)
- @param limit: iterate no more than this number of entries
- @param namespaces: iterate changes to pages in these namespaces only
- @type namespaces: list of ints
- @param pagelist: iterate changes to pages in this list only
- @param pagelist: list of Pages
- @param changetype: only iterate changes of this type ("edit" for
- edits to existing pages, "new" for new pages, "log" for log
- entries)
- @param showMinor: if True, only list minor edits; if False (and not
- None), only list non-minor edits
- @param showBot: if True, only list bot edits; if False (and not
- None), only list non-bot edits
- @param showAnon: if True, only list anon edits; if False (and not
- None), only list non-anon edits
- @param showRedirects: if True, only list edits to redirect pages; if
- False (and not None), only list edits to non-redirect pages
- @param showPatrolled: if True, only list patrolled edits; if False
- (and not None), only list non-patrolled edits
-
- """
- if start and end:
- if reverse:
- if end < start:
- raise Error(
- "recentchanges: end must be later than start with reverse=True")
- else:
- if start < end:
- raise Error(
- "recentchanges: start must be later than end with reverse=False")
- rcgen = api.ListGenerator("recentchanges", site=self,
- rcprop="user|comment|timestamp|title|ids"
- "|redirect|patrolled|loginfo|flags")
- if start is not None:
- rcgen.request["rcstart"] = start
- if end is not None:
- rcgen.request["rcend"] = end
- if reverse:
- rcgen.request["rcdir"] = "newer"
- if isinstance(limit, int):
- rcgen.limit = limit
- if isinstance(namespaces, list):
- rcgen.request["rcnamespace"] = u"|".join(unicode(ns)
- for ns in namespaces)
- elif namespaces is not None:
- rcgen.request["rcnamespace"] = str(namespaces)
- if pagelist:
- rcgen.request["rctitles"] = u"|".join(p.title(withSection=False)
- for p in pagelist)
- if changetype:
- rcgen.request["rctype"] = changetype
- filters = {'minor': showMinor,
- 'bot': showBot,
- 'anon': showAnon,
- 'redirect': showRedirects,
- 'patrolled': showPatrolled}
- rcshow = []
- for item in filters:
- if filters[item] is not None:
- rcshow.append(filters[item] and item or ("!"+item))
- if rcshow:
- rcgen.request["rcshow"] = "|".join(rcshow)
- return rcgen
-
- @deprecate_arg("number", "limit")
- def search(self, searchstring, namespaces=None, where="text",
- getredirects=False, limit=None):
- """Iterate Pages that contain the searchstring.
-
- Note that this may include non-existing Pages if the wiki's database
- table contains outdated entries.
-
- @param searchstring: the text to search for
- @type searchstring: unicode
- @param where: Where to search; value must be "text" or "titles" (many
- wikis do not support title search)
- @param namespaces: search only in these namespaces (defaults to 0)
- @type namespaces: list of ints
- @param getredirects: if True, include redirects in results
- @param limit: maximum number of results to iterate
-
- """
- if not searchstring:
- raise Error("search: searchstring cannot be empty")
- if where not in ("text", "titles"):
- raise Error("search: unrecognized 'where' value: %s" % where)
- srgen = api.PageGenerator("search", gsrsearch=searchstring,
- gsrwhat=where, site=self)
- if not namespaces:
- logger.warning("search: namespaces cannot be empty; using [0].")
- namespaces = [0]
- if isinstance(namespaces, list):
- srgen.request["gsrnamespace"] = u"|".join(unicode(ns)
- for ns in namespaces)
- else:
- srgen.request["gsrnamespace"] = str(namespaces)
- if getredirects:
- srgen.request["gsrredirects"] = ""
- if isinstance(limit, int):
- srgen.limit = limit
- return srgen
-
- def usercontribs(self, user=None, userprefix=None, start=None, end=None,
- reverse=False, limit=None, namespaces=None,
- showMinor=None):
- """Iterate contributions by a particular user.
-
- Iterated values are in the same format as recentchanges.
-
- @param user: Iterate contributions by this user (name or IP)
- @param userprefix: Iterate contributions by all users whose names
- or IPs start with this substring
- @param start: Iterate contributions starting at this timestamp
- @param end: Iterate contributions ending at this timestamp
- @param reverse: Iterate oldest contributions first (default: newest)
- @param limit: Maximum number of contributions to iterate
- @param namespaces: Only iterate contributions in these namespaces
- @type namespaces: list of ints
- @param showMinor: if True, iterate only minor edits; if False and
- not None, iterate only non-minor edits (default: iterate both)
-
- """
- if not (user or userprefix):
- raise Error(
- "usercontribs: either user or userprefix must be non-empty")
- if start and end:
- if reverse:
- if end < start:
- raise Error(
- "usercontribs: end must be later than start with reverse=True")
- else:
- if start < end:
- raise Error(
- "usercontribs: start must be later than end with reverse=False")
- ucgen = api.ListGenerator("usercontribs", site=self,
- ucprop="ids|title|timestamp|comment|flags")
- if user:
- ucgen.request["ucuser"] = user
- if userprefix:
- ucgen.request["ucuserprefix"] = userprefix
- if start is not None:
- ucgen.request["ucstart"] = start
- if end is not None:
- ucgen.request["ucend"] = end
- if reverse:
- ucgen.request["ucdir"] = "newer"
- if isinstance(limit, int):
- ucgen.limit = limit
- if isinstance(namespaces, list):
- ucgen.request["ucnamespace"] = u"|".join(unicode(ns)
- for ns in namespaces)
- elif namespaces is not None:
- ucgen.request["ucnamespace"] = str(namespaces)
- if showMinor is not None:
- ucgen.request["ucshow"] = showMinor and "minor" or "!minor"
- return ucgen
-
- def watchlist_revs(self, start=None, end=None, reverse=False,
- namespaces=None, showMinor=None, showBot=None,
- showAnon=None, limit=None):
- """Iterate revisions to pages on the bot user's watchlist.
-
- Iterated values will be in same format as recentchanges.
-
- @param start: Iterate revisions starting at this timestamp
- @param end: Iterate revisions ending at this timestamp
- @param reverse: Iterate oldest revisions first (default: newest)
- @param namespaces: only iterate revisions to pages in these
- namespaces (default: all)
- @type namespaces: list of ints
- @param showMinor: if True, only list minor edits; if False (and not
- None), only list non-minor edits
- @param showBot: if True, only list bot edits; if False (and not
- None), only list non-bot edits
- @param showAnon: if True, only list anon edits; if False (and not
- None), only list non-anon edits
- @param limit: Maximum number of revisions to iterate
-
- """
- if start and end:
- if reverse:
- if end < start:
- raise Error(
- "watchlist_revs: end must be later than start with reverse=True")
- else:
- if start < end:
- raise Error(
- "watchlist_revs: start must be later than end with reverse=False")
- wlgen = api.ListGenerator("watchlist", wlallrev="", site=self,
- wlprop="user|comment|timestamp|title|ids|flags")
- #TODO: allow users to ask for "patrol" as well?
- if start is not None:
- wlgen.request["wlstart"] = start
- if end is not None:
- wlgen.request["wlend"] = end
- if reverse:
- wlgen.request["wldir"] = "newer"
- if isinstance(limit, int):
- wlgen.limit = limit
- if isinstance(namespaces, list):
- wlgen.request["wlnamespace"] = u"|".join(unicode(ns)
- for ns in namespaces)
- elif namespaces is not None:
- wlgen.request["wlnamespace"] = str(namespaces)
- filters = {'minor': showMinor,
- 'bot': showBot,
- 'anon': showAnon}
- wlshow = []
- for item in filters:
- if filters[item] is not None:
- wlshow.append(filters[item] and item or ("!"+item))
- if wlshow:
- wlgen.request["wlshow"] = "|".join(wlshow)
- return wlgen
-
- def deletedrevs(self, page, start=None, end=None, reverse=None, limit=None,
- get_text=False):
- """Iterate deleted revisions.
-
- Each value returned by the iterator will be a dict containing the
- 'title' and 'ns' keys for a particular Page and a 'revisions' key
- whose value is a list of revisions in the same format as
- recentchanges (plus a 'content' element if requested). If get_text
- is true, the toplevel dict will contain a 'token' key as well.
-
- @param page: The page to check for deleted revisions
- @param start: Iterate revisions starting at this timestamp
- @param end: Iterate revisions ending at this timestamp
- @param reverse: Iterate oldest revisions first (default: newest)
- @param limit: Iterate no more than this number of revisions.
- @param get_text: If True, retrieve the content of each revision and
- an undelete token
-
- """
- if start and end:
- if reverse:
- if end < start:
- raise Error(
-"deletedrevs: end must be later than start with reverse=True")
- else:
- if start < end:
- raise Error(
-"deletedrevs: start must be later than end with reverse=False")
- if not self.logged_in():
- self.login()
- if "deletedhistory" not in self.userinfo['rights']:
- try:
- self.login(True)
- except NoUsername:
- pass
- if "deletedhistory" not in self.userinfo['rights']:
- raise Error(
-"deletedrevs: User:%s not authorized to access deleted revisions."
- % self.user())
- if get_text:
- if "undelete" not in self.userinfo['rights']:
- try:
- self.login(True)
- except NoUsername:
- pass
- if "undelete" not in self.userinfo['rights']:
- raise Error(
-"deletedrevs: User:%s not authorized to view deleted content."
- % self.user())
-
- drgen = api.ListGenerator("deletedrevs", site=self,
- titles=page.title(withSection=False),
- drprop="revid|user|comment|minor")
- if get_text:
- drgen.request['drprop'] = drgen.request['drprop'] + "|content|token"
- if start is not None:
- drgen.request["drstart"] = start
- if end is not None:
- drgen.request["drend"] = end
- if reverse:
- drgen.request["drdir"] = "newer"
- if isinstance(limit, int):
- drgen.limit = limit
- return drgen
-
- def users(self, usernames):
- """Iterate info about a list of users by name or IP.
-
- @param usernames: a list of user names
- @type usernames: list, or other iterable, of unicodes
-
- """
- if not isinstance(usernames, basestring):
- usernames = u"|".join(usernames)
- usgen = api.ListGenerator("users", ususers=usernames, site=self,
- usprop="blockinfo|groups|editcount|registration")
- return usgen
-
- def randompages(self, limit=1, namespaces=None, redirects=False):
- """Iterate a number of random pages.
-
- Pages are listed in a fixed sequence, only the starting point is
- random.
-
- @param limit: the maximum number of pages to iterate (default: 1)
- @param namespaces: only iterate pages in these namespaces.
- @param redirects: if True, include only redirect pages in results
- (default: include only non-redirects)
-
- """
- rngen = api.PageGenerator("random", site=self)
- rngen.limit = limit
- if isinstance(namespaces, list):
- rngen.request["grnnamespace"] = u"|".join(unicode(ns)
- for ns in namespaces)
- elif namespaces is not None:
- rngen.request["grnnamespace"] = str(namespaces)
- if redirects:
- rngen.request["grnredirect"] = ""
- return rngen
-
- # catalog of editpage error codes, for use in generating messages
- _ep_errors = {
- "noapiwrite": "API editing not enabled on %(site)s wiki",
- "writeapidenied":
-"User %(user)s is not authorized to edit on %(site)s wiki",
- "protectedtitle":
-"Title %(title)s is protected against creation on %(site)s",
- "cantcreate":
-"User %(user)s not authorized to create new pages on %(site)s wiki",
- "cantcreate-anon":
-"""Bot is not logged in, and anon users are not authorized to create new pages
-on %(site)s wiki""",
- "articleexists": "Page %(title)s already exists on %(site)s wiki",
- "noimageredirect-anon":
-"""Bot is not logged in, and anon users are not authorized to create image
-redirects on %(site)s wiki""",
- "noimageredirect":
-"User %(user)s not authorized to create image redirects on %(site)s wiki",
- "spamdetected":
-"Edit to page %(title)s rejected by spam filter due to content:\n",
- "filtered": "%(info)s",
- "contenttoobig": "%(info)s",
- "noedit-anon":
-"""Bot is not logged in, and anon users are not authorized to edit on
-%(site)s wiki""",
- "noedit": "User %(user)s not authorized to edit pages on %(site)s wiki",
- "pagedeleted":
-"Page %(title)s has been deleted since last retrieved from %(site)s wiki",
- "editconflict": "Page %(title)s not saved due to edit conflict.",
- }
-
- def editpage(self, page, summary, minor=True, notminor=False,
- recreate=True, createonly=False, watch=False, unwatch=False):
- """Submit an edited Page object to be saved to the wiki.
-
- @param page: The Page to be saved; its .text property will be used
- as the new text to be saved to the wiki
- @param token: the edit token retrieved using Site.token()
- @param summary: the edit summary (required!)
- @param minor: if True (default), mark edit as minor
- @param notminor: if True, override account preferences to mark edit
- as non-minor
- @param recreate: if True (default), create new page even if this
- title has previously been deleted
- @param createonly: if True, raise an error if this title already
- exists on the wiki
- @param watch: if True, add this Page to bot's watchlist
- @param unwatch: if True, remove this Page from bot's watchlist if
- possible
- @return: True if edit succeeded, False if it failed
-
- """
- text = page.text
- if not text:
- raise Error("editpage: no text to be saved")
- try:
- lastrev = page.latestRevision()
- except NoPage:
- lastrev = None
- if not recreate:
- raise Error("Page %s does not exist on %s wiki."
- % (page.title(withSection=False), self))
- token = self.token(page, "edit")
- self.lock_page(page)
- if lastrev is not None and page.latestRevision() != lastrev:
- raise Error("editpage: Edit conflict detected; saving aborted.")
- req = api.Request(site=self, action="edit",
- title=page.title(withSection=False),
- text=text, token=token, summary=summary)
-## if lastrev is not None:
-## req["basetimestamp"] = page._revisions[lastrev].timestamp
- if minor:
- req['minor'] = ""
- elif notminor:
- req['notminor'] = ""
- if 'bot' in self.userinfo['groups']:
- req['bot'] = ""
- if recreate:
- req['recreate'] = ""
- if createonly:
- req['createonly'] = ""
- if watch:
- req['watch'] = ""
- elif unwatch:
- req['unwatch'] = ""
-## FIXME: API gives 'badmd5' error
-## md5hash = md5()
-## md5hash.update(urllib.quote_plus(text.encode(self.encoding())))
-## req['md5'] = md5hash.digest()
- while True:
- try:
- result = req.submit()
- logger.debug("editpage response: %s" % result)
- except api.APIError, err:
- self.unlock_page(page)
- if err.code.endswith("anon") and self.logged_in():
- logger.debug(
-"editpage: received '%s' even though bot is logged in" % err.code)
- errdata = {
- 'site': self,
- 'title': page.title(withSection=False),
- 'user': self.user(),
- 'info': err.info
- }
- if err.code == "spamdetected":
- raise SpamfilterError(self._ep_errors[err.code] % errdata
- + err.info[ err.info.index("fragment: ") + 9: ])
-
- if err.code == "editconflict":
- raise EditConflict(self._ep_errors[err.code] % errdata)
- if err.code in self._ep_errors:
- raise Error(self._ep_errors[err.code] % errdata)
- logger.debug("editpage: Unexpected error code '%s' received."
- % err.code)
- raise
- assert ("edit" in result and "result" in result["edit"]), result
- if result["edit"]["result"] == "Success":
- self.unlock_page(page)
- if "nochange" in result["edit"]:
- # null edit, page not changed
- # TODO: do we want to notify the user of this?
- return True
- page._revid = result["edit"]["newrevid"]
- # see http://www.mediawiki.org/wiki/API:Wikimania_2006_API_discussion#Notes
- # not safe to assume that saved text is the same as sent
- self.loadrevisions(page, getText=True)
- return True
- elif result["edit"]["result"] == "Failure":
- if "captcha" in result["edit"]:
- captcha = result["edit"]["captcha"]
- req['captchaid'] = captcha['id']
- if captcha["type"] == "math":
- req['captchaword'] = input(captcha["question"])
- continue
- elif "url" in captcha:
- webbrowser.open(url)
- req['captchaword'] = cap_answerwikipedia.input(
-"Please view CAPTCHA in your browser, then type answer here:")
- continue
- else:
- self.unlock_page(page)
- logger.error(
-"editpage: unknown CAPTCHA response %s, page not saved"
- % captcha)
- return False
- else:
- self.unlock_page(page)
- logger.error("editpage: unknown failure reason %s"
- % str(result))
- return False
- else:
- self.unlock_page(page)
- logger.error(
-"editpage: Unknown result code '%s' received; page not saved"
- % result["edit"]["result"])
- logger.error(str(result))
- return False
-
- # catalog of move errors for use in error messages
- _mv_errors = {
- "noapiwrite": "API editing not enabled on %(site)s wiki",
- "writeapidenied":
-"User %(user)s is not authorized to edit on %(site)s wiki",
- "nosuppress":
-"User %(user)s is not authorized to move pages without creating redirects",
- "cantmove-anon":
-"""Bot is not logged in, and anon users are not authorized to move pages on
-%(site)s wiki""",
- "cantmove":
-"User %(user)s is not authorized to move pages on %(site)s wiki",
- "immobilenamespace":
-"Pages in %(oldnamespace)s namespace cannot be moved on %(site)s wiki",
- "articleexists":
-"Cannot move because page [[%(newtitle)s]] already exists on %(site)s wiki",
- "protectedpage":
-"Page [[%(oldtitle)s]] is protected against moving on %(site)s wiki",
- "protectedtitle":
-"Page [[%(newtitle)s]] is protected against creation on %(site)s wiki",
- "nonfilenamespace":
-"Cannot move a file to %(newnamespace)s namespace on %(site)s wiki",
- "filetypemismatch":
-"[[%(newtitle)s]] file extension does not match content of [[%(oldtitle)s]]"
- }
-
- def movepage(self, page, newtitle, summary, movetalk=True,
- noredirect=False):
- """Move a Page to a new title.
-
- @param page: the Page to be moved (must exist)
- @param newtitle: the new title for the Page
- @type newtitle: unicode
- @param summary: edit summary (required!)
- @param movetalk: if True (default), also move the talk page if possible
- @param noredirect: if True, suppress creation of a redirect from the
- old title to the new one
- @return: Page object with the new title
-
- """
- oldtitle = page.title(withSection=False)
- newlink = pywikibot.Link(newtitle, self)
- if newlink.namespace:
- newtitle = self.namespace(newlink.namespace) + ":" + newlink.title
- else:
- newtitle = newlink.title
- if oldtitle == newtitle:
- raise Error("Cannot move page %s to its own title."
- % oldtitle)
- if not page.exists():
- raise Error("Cannot move page %s because it does not exist on %s."
- % (oldtitle, self))
- token = self.token(page, "move")
- self.lock_page(page)
- req = api.Request(site=self, action="move", to=newtitle,
- token=token, reason=summary)
- req['from'] = oldtitle # "from" is a python keyword
- if movetalk:
- req['movetalk'] = ""
- if noredirect:
- req['noredirect'] = ""
- try:
- result = req.submit()
- logger.debug("movepage response: %s" % result)
- except api.APIError, err:
- if err.code.endswith("anon") and self.logged_in():
- logger.debug(
-"movepage: received '%s' even though bot is logged in" % err.code)
- errdata = {
- 'site': self,
- 'oldtitle': oldtitle,
- 'oldnamespace': self.namespace(page.namespace()),
- 'newtitle': newtitle,
- 'newnamespace': self.namespace(newlink.namespace),
- 'user': self.user(),
- }
- if err.code in self._mv_errors:
- raise Error(self._mv_errors[err.code] % errdata)
- logger.debug("movepage: Unexpected error code '%s' received."
- % err.code)
- raise
- finally:
- self.unlock_page(page)
- if "move" not in result:
- logger.error("movepage: %s" % result)
- raise Error("movepage: unexpected response")
- # TODO: Check for talkmove-error messages
- if "talkmove-error-code" in result["move"]:
- logger.warning(u"movepage: Talk page %s not moved"
- % (page.toggleTalkPage().title(asLink=True)))
- return pywikibot.Page(page, newtitle)
-
- # catalog of rollback errors for use in error messages
- _rb_errors = {
- "noapiwrite":
- "API editing not enabled on %(site)s wiki",
- "writeapidenied":
- "User %(user)s not allowed to edit through the API",
- "alreadyrolled":
- "Page [[%(title)s]] already rolled back; action aborted.",
- } # other errors shouldn't arise because we check for those errors
-
- def rollbackpage(self, page, summary=u''):
- """Roll back page to version before last user's edits.
-
- As a precaution against errors, this method will fail unless
- the page history contains at least two revisions, and at least
- one that is not by the same user who made the last edit.
-
- @param page: the Page to be rolled back (must exist)
- @param summary: edit summary (defaults to a standardized message)
-
- """
- if len(page._revisions) < 2:
- raise pywikibot.Error(
- u"Rollback of %s aborted; load revision history first."
- % page.title(asLink=True))
- last_rev = page._revisions[page.latestRevision()]
- last_user = last_rev.user
- for rev in sorted(page._revisions.keys(), reverse=True):
- # start with most recent revision first
- if rev.user != last_user:
- prev_user = rev.user
- break
- else:
- raise pywikibot.Error(
- u"Rollback of %s aborted; only one user in revision history."
- % page.title(asLink=True))
- summary = summary or (
-u"Reverted edits by [[Special:Contributions/%(last_user)s|%(last_user)s]] "
-u"([[User talk:%(last_user)s|Talk]]) to last version by %(prev_user)s"
- % locals())
- token = self.token(page, "rollback")
- self.lock_page(page)
- req = api.Request(site=self, action="rollback",
- title=page.title(withSection=False),
- user=last_user,
- token=token)
- try:
- result = req.submit()
- except api.APIError, err:
- errdata = {
- 'site': self,
- 'title': page.title(withSection=False),
- 'user': self.user(),
- }
- if err.code in self._rb_errors:
- raise Error(self._rb_errors[err.code] % errdata)
- logger.debug("rollback: Unexpected error code '%s' received."
- % err.code)
- raise
- finally:
- self.unlock_page(page)
-
- # catalog of delete errors for use in error messages
- _dl_errors = {
- "noapiwrite":
- "API editing not enabled on %(site)s wiki",
- "writeapidenied":
- "User %(user)s not allowed to edit through the API",
- "permissiondenied":
- "User %(user)s not authorized to delete pages on %(site)s wiki.",
- "cantdelete":
- "Could not delete [[%(title)s]]. Maybe it was deleted already.",
- } # other errors shouldn't occur because of pre-submission checks
-
- def deletepage(self, page, summary):
- """Delete page from the wiki. Requires appropriate privilege level.
-
- @param page: Page to be deleted.
- @param summary: Edit summary (required!).
-
- """
- try:
- self.login(sysop=True)
- except pywikibot.Error, e:
- raise Error("delete: Unable to login as sysop (%s)"
- % e.__class__.__name__)
- if not self.logged_in(sysop=True):
- raise Error("delete: Unable to login as sysop")
- token = self.token("delete")
- req = api.Request(site=self, action="delete", token=token,
- title=page.title(withSection=False),
- reason=summary)
- try:
- result = req.submit()
- except api.APIError, err:
- errdata = {
- 'site': self,
- 'title': page.title(withSection=False),
- 'user': self.user(),
- }
- if err.code in self._dl_errors:
- raise Error(self._dl_errors[err.code] % errdata)
- logger.debug("delete: Unexpected error code '%s' received."
- % err.code)
- raise
- finally:
- self.unlock_page(page)
-
- # TODO: implement undelete
-
- # TODO: implement patrol
-
- def linksearch(self, siteurl, limit=500):
- """Backwards-compatible interface to exturlusage()"""
- return self.exturlusage(siteurl, limit=limit)
-
- @deprecate_arg("repeat", None)
- def newimages(self, number=100, lestart=None, leend=None, leuser=None,
- letitle=None):
- """Yield ImagePages from most recent uploads"""
- return self.logevents(logtype="upload", limit=number, start=lestart,
- end=leend, user=leuser, title=letitle)
-
- def getImagesFromAnHash(self, hash_found=None):
- """Return all images that have the same hash.
-
- Useful to find duplicates or nowcommons.
-
- NOTE: it returns also the image itself, if you don't want it, just
- filter the list returned.
-
- NOTE 2: it returns the image title WITHOUT the image namespace.
-
- """
- if hash_found == None: # If the hash is none return None and not continue
- return None
- return [image.title(withNamespace=False)
- for image in self.allimages(sha1=hash_found)]
-
-
-#### METHODS NOT IMPLEMENTED YET ####
-class NotImplementedYet:
-
- # TODO: is this needed any more? can it be obtained from the http module?
- def cookies(self, sysop = False):
- """Return a string containing the user's current cookies."""
- self._loadCookies(sysop = sysop)
- index = self._userIndex(sysop)
- return self._cookies[index]
-
- def _loadCookies(self, sysop = False):
- """Retrieve session cookies for login"""
- index = self._userIndex(sysop)
- if self._cookies[index] is not None:
- return
- try:
- if sysop:
- try:
- username = config.sysopnames[self.family.name
- ][self.code]
- except KeyError:
- raise NoUsername("""\
-You tried to perform an action that requires admin privileges, but you haven't
-entered your sysop name in your user-config.py. Please add
-sysopnames['%s']['%s']='name' to your user-config.py"""
- % (self.family.name, self.code))
- else:
- username = pywikiobt.config2.usernames[self.family.name
- ][self.code]
- except KeyError:
- self._cookies[index] = None
- self._isLoggedIn[index] = False
- else:
- tmp = '%s-%s-%s-login.data' % (
- self.family.name, self.code, username)
- fn = config.datafilepath('login-data', tmp)
- if not os.path.exists(fn):
- self._cookies[index] = None
- self._isLoggedIn[index] = False
- else:
- f = open(fn)
- self._cookies[index] = '; '.join([x.strip() for x in f.readlines()])
- f.close()
-
- # THESE ARE FUNCTIONS NOT YET IMPLEMENTED IN THE API
- # TODO: avoid code duplication for the following methods
- def newpages(self, number = 10, get_redirect = False, repeat = False):
- """Yield new articles (as Page objects) from Special:Newpages.
-
- Starts with the newest article and fetches the number of articles
- specified in the first argument. If repeat is True, it fetches
- Newpages again. If there is no new page, it blocks until there is
- one, sleeping between subsequent fetches of Newpages.
-
- The objects yielded are tuples composed of the Page object,
- timestamp (unicode), length (int), an empty unicode string, username
- or IP address (str), comment (unicode).
-
- """
- # TODO: in recent MW versions Special:Newpages takes a namespace parameter,
- # and defaults to 0 if not specified.
- # TODO: Detection of unregistered users is broken
- # TODO: Repeat mechanism doesn't make much sense as implemented;
- # should use both offset and limit parameters, and have an
- # option to fetch older rather than newer pages
- seen = set()
- while True:
- path = self.newpages_address(n=number)
- # The throttling is important here, so always enabled.
- get_throttle()
- html = self.getUrl(path)
-
- entryR = re.compile(
-'<li[^>]*>(?P<date>.+?) \S*?<a href=".+?"'
-' title="(?P<title>.+?)">.+?</a>.+?[\(\[](?P<length>[\d,.]+)[^\)\]]*[\)\]]'
-' .?<a href=".+?" title=".+?:(?P<username>.+?)">'
- )
- for m in entryR.finditer(html):
- date = m.group('date')
- title = m.group('title')
- title = title.replace('"', '"')
- length = int(re.sub("[,.]", "", m.group('length')))
- loggedIn = u''
- username = m.group('username')
- comment = u''
-
- if title not in seen:
- seen.add(title)
- page = Page(self, title)
- yield page, date, length, loggedIn, username, comment
- if not repeat:
- break
-
- def longpages(self, number = 10, repeat = False):
- """Yield Pages from Special:Longpages.
-
- Return values are a tuple of Page object, length(int).
-
- """
- #TODO: should use offset and limit parameters; 'repeat' as now
- # implemented is fairly useless
- # this comment applies to all the XXXXpages methods following, as well
- seen = set()
- while True:
- path = self.longpages_address(n=number)
- get_throttle()
- html = self.getUrl(path)
- entryR = re.compile(ur'<li>\(<a href=".+?" title=".+?">hist</a>\) <a href=".+?" title="(?P<title>.+?)">.+?</a> \[(?P<length>\d+)(.+?)\]</li>')
- for m in entryR.finditer(html):
- title = m.group('title')
- length = int(m.group('length'))
- if title not in seen:
- seen.add(title)
- page = Page(self, title)
- yield page, length
- if not repeat:
- break
-
- def shortpages(self, number = 10, repeat = False):
- """Yield Pages and lengths from Special:Shortpages."""
- throttle = True
- seen = set()
- while True:
- path = self.shortpages_address(n = number)
- get_throttle()
- html = self.getUrl(path)
- entryR = re.compile(ur'<li>\(<a href=".+?" title=".+?">hist</a>\) <a href=".+?" title="(?P<title>.+?)">.+?</a> \[(?P<length>\d+)(.+?)\]</li>')
- for m in entryR.finditer(html):
- title = m.group('title')
- length = int(m.group('length'))
-
- if title not in seen:
- seen.add(title)
- page = Page(self, title)
- yield page, length
- if not repeat:
- break
-
- def deadendpages(self, number = 10, repeat = False):
- """Yield Page objects retrieved from Special:Deadendpages."""
- seen = set()
- while True:
- path = self.deadendpages_address(n=number)
- get_throttle()
- html = self.getUrl(path)
- entryR = re.compile(
- '<li><a href=".+?" title="(?P<title>.+?)">.+?</a></li>')
- for m in entryR.finditer(html):
- title = m.group('title')
-
- if title not in seen:
- seen.add(title)
- page = Page(self, title)
- yield page
- if not repeat:
- break
-
- def ancientpages(self, number = 10, repeat = False):
- """Yield Pages, datestamps from Special:Ancientpages."""
- seen = set()
- while True:
- path = self.ancientpages_address(n=number)
- get_throttle()
- html = self.getUrl(path)
- entryR = re.compile(
-'<li><a href=".+?" title="(?P<title>.+?)">.+?</a> (?P<date>.+?)</li>')
- for m in entryR.finditer(html):
- title = m.group('title')
- date = m.group('date')
- if title not in seen:
- seen.add(title)
- page = Page(self, title)
- yield page, date
- if not repeat:
- break
-
- def lonelypages(self, number = 10, repeat = False):
- """Yield Pages retrieved from Special:Lonelypages."""
- throttle = True
- seen = set()
- while True:
- path = self.lonelypages_address(n=number)
- get_throttle()
- html = self.getUrl(path)
- entryR = re.compile(
- '<li><a href=".+?" title="(?P<title>.+?)">.+?</a></li>')
- for m in entryR.finditer(html):
- title = m.group('title')
-
- if title not in seen:
- seen.add(title)
- page = Page(self, title)
- yield page
- if not repeat:
- break
-
- def unwatchedpages(self, number = 10, repeat = False):
- """Yield Pages from Special:Unwatchedpages (requires Admin privileges)."""
- seen = set()
- while True:
- path = self.unwatchedpages_address(n=number)
- get_throttle()
- html = self.getUrl(path, sysop = True)
- entryR = re.compile(
- '<li><a href=".+?" title="(?P<title>.+?)">.+?</a>.+?</li>')
- for m in entryR.finditer(html):
- title = m.group('title')
- if title not in seen:
- seen.add(title)
- page = Page(self, title)
- yield page
- if not repeat:
- break
-
- def uncategorizedcategories(self, number = 10, repeat = False):
- """Yield Categories from Special:Uncategorizedcategories."""
- import catlib
- seen = set()
- while True:
- path = self.uncategorizedcategories_address(n=number)
- get_throttle()
- html = self.getUrl(path)
- entryR = re.compile(
- '<li><a href=".+?" title="(?P<title>.+?)">.+?</a></li>')
- for m in entryR.finditer(html):
- title = m.group('title')
- if title not in seen:
- seen.add(title)
- page = catlib.Category(self, title)
- yield page
- if not repeat:
- break
-
- def newimages(self, number = 10, repeat = False):
- """Yield ImagePages from Special:Log&type=upload"""
-
- seen = set()
- regexp = re.compile('<li[^>]*>(?P<date>.+?)\s+<a href=.*?>(?P<user>.+?)</a>\s+\(.+?</a>\).*?<a href=".*?"(?P<new> class="new")? title="(?P<image>.+?)"\s*>(?:.*?<span class="comment">(?P<comment>.*?)</span>)?', re.UNICODE)
-
- while True:
- path = self.log_address(number, mode = 'upload')
- get_throttle()
- html = self.getUrl(path)
-
- for m in regexp.finditer(html):
- image = m.group('image')
-
- if image not in seen:
- seen.add(image)
-
- if m.group('new'):
- output(u"Image \'%s\' has been deleted." % image)
- continue
-
- date = m.group('date')
- user = m.group('user')
- comment = m.group('comment') or ''
-
- yield ImagePage(self, image), date, user, comment
- if not repeat:
- break
-
- def uncategorizedimages(self, number = 10, repeat = False):
- """Yield ImagePages from Special:Uncategorizedimages."""
- seen = set()
- ns = self.image_namespace()
- entryR = re.compile(
- '<a href=".+?" title="(?P<title>%s:.+?)">.+?</a>' % ns)
- while True:
- path = self.uncategorizedimages_address(n=number)
- get_throttle()
- html = self.getUrl(path)
- for m in entryR.finditer(html):
- title = m.group('title')
- if title not in seen:
- seen.add(title)
- page = ImagePage(self, title)
- yield page
- if not repeat:
- break
-
- def uncategorizedpages(self, number = 10, repeat = False):
- """Yield Pages from Special:Uncategorizedpages."""
- seen = set()
- while True:
- path = self.uncategorizedpages_address(n=number)
- get_throttle()
- html = self.getUrl(path)
- entryR = re.compile(
- '<li><a href=".+?" title="(?P<title>.+?)">.+?</a></li>')
- for m in entryR.finditer(html):
- title = m.group('title')
-
- if title not in seen:
- seen.add(title)
- page = Page(self, title)
- yield page
- if not repeat:
- break
-
- def unusedcategories(self, number = 10, repeat = False):
- """Yield Category objects from Special:Unusedcategories."""
- import catlib
- seen = set()
- while True:
- path = self.unusedcategories_address(n=number)
- get_throttle()
- html = self.getUrl(path)
- entryR = re.compile('<li><a href=".+?" title="(?P<title>.+?)">.+?</a></li>')
- for m in entryR.finditer(html):
- title = m.group('title')
-
- if title not in seen:
- seen.add(title)
- page = catlib.Category(self, title)
- yield page
- if not repeat:
- break
-
- def unusedfiles(self, number = 10, repeat = False, extension = None):
- """Yield ImagePage objects from Special:Unusedimages."""
- seen = set()
- ns = self.image_namespace()
- entryR = re.compile(
- '<a href=".+?" title="(?P<title>%s:.+?)">.+?</a>' % ns)
- while True:
- path = self.unusedfiles_address(n=number)
- get_throttle()
- html = self.getUrl(path)
- for m in entryR.finditer(html):
- fileext = None
- title = m.group('title')
- if extension:
- fileext = title[len(title)-3:]
- if title not in seen and fileext == extension:
- ## Check whether the media is used in a Proofread page
- # code disabled because it slows this method down, and
- # because it is unclear what it's supposed to do.
- #basename = title[6:]
- #page = Page(self, 'Page:' + basename)
-
- #if not page.exists():
- seen.add(title)
- image = ImagePage(self, title)
- yield image
- if not repeat:
- break
-
- def withoutinterwiki(self, number=10, repeat=False):
- """Yield Pages without language links from Special:Withoutinterwiki."""
- seen = set()
- while True:
- path = self.withoutinterwiki_address(n=number)
- get_throttle()
- html = self.getUrl(path)
- entryR = re.compile('<li><a href=".+?" title="(?P<title>.+?)">.+?</a></li>')
- for m in entryR.finditer(html):
- title = m.group('title')
- if title not in seen:
- seen.add(title)
- page = Page(self, title)
- yield page
- if not repeat:
- break
-
- def linksearch(self, siteurl):
- """Yield Pages from results of Special:Linksearch for 'siteurl'."""
- if siteurl.startswith('*.'):
- siteurl = siteurl[2:]
- output(u'Querying [[Special:Linksearch]]...')
- cache = []
- for url in [siteurl, '*.' + siteurl]:
- path = self.linksearch_address(url)
- get_throttle()
- html = self.getUrl(path)
- loc = html.find('<div class="mw-spcontent">')
- if loc > -1:
- html = html[loc:]
- loc = html.find('<div class="printfooter">')
- if loc > -1:
- html = html[:loc]
- R = re.compile('title ?=\"(.*?)\"')
- for title in R.findall(html):
- if not siteurl in title:
- # the links themselves have similar form
- if title in cache:
- continue
- else:
- cache.append(title)
- yield Page(self, title)
-
+ # -*- coding: utf-8 -*-
+"""
+Objects representing MediaWiki sites (wikis) and families (groups of wikis
+on the same topic in different languages).
+"""
+#
+# (C) Pywikipedia bot team, 2008
+#
+# Distributed under the terms of the MIT license.
+#
+__version__ = '$Id$'
+
+import pywikibot
+from pywikibot import deprecate_arg
+from pywikibot import config
+from pywikibot.throttle import Throttle
+from pywikibot.data import api
+from pywikibot.exceptions import *
+
+try:
+ from hashlib import md5
+except ImportError:
+ from md5 import md5
+import logging
+import os
+import re
+import sys
+import threading
+import urllib
+
+logger = logging.getLogger("wiki")
+
+class PageInUse(pywikibot.Error):
+ """Page cannot be reserved for writing due to existing lock."""
+
+
+def Family(fam=None, fatal=True):
+ """Import the named family.
+
+ @param fam: family name (if omitted, uses the configured default)
+ @type fam: str
+ @param fatal: if True, the bot will stop running if the given family is
+ unknown. If False, it will only raise a ValueError exception.
+ @param fatal: bool
+ @return: a Family instance configured for the named family.
+
+ """
+ if fam == None:
+ fam = config.family
+ try:
+ # first try the built-in families
+ exec "import pywikibot.families.%s_family as myfamily" % fam
+ except ImportError:
+ # next see if user has defined a local family module
+ try:
+ sys.path.append(config.datafilepath('families'))
+ exec "import %s_family as myfamily" % fam
+ except ImportError:
+ if fatal:
+ logger.exception(u"""\
+Error importing the %s family. This probably means the family
+does not exist. Also check your configuration file."""
+ % fam)
+ sys.exit(1)
+ else:
+ raise Error("Family %s does not exist" % fam)
+ return myfamily.Family()
+
+
+class BaseSite(object):
+ """Site methods that are independent of the communication interface."""
+ # to implement a specific interface, define a Site class that inherits
+ # from this
+
+ def __init__(self, code, fam=None, user=None, sysop=None):
+ """
+ @param code: the site's language code
+ @type code: str
+ @param fam: wiki family name (optional)
+ @type fam: str or Family
+ @param user: bot user name (optional)
+ @type user: str
+ @param sysop: sysop account user name (optional)
+ @type sysop: str
+
+ """
+ self.__code = code.lower()
+ if isinstance(fam, basestring) or fam is None:
+ self.__family = Family(fam, fatal=False)
+ else:
+ self.__family = fam
+
+ # if we got an outdated language code, use the new one instead.
+ if self.__family.obsolete.has_key(self.__code):
+ if self.__family.obsolete[self.__code] is not None:
+ self.__code = self.__family.obsolete[self.__code]
+ else:
+ # no such language anymore
+ raise NoSuchSite("Language %s in family %s is obsolete"
+ % (self.__code, self.__family.name))
+ if self.__code not in self.languages():
+ if self.__code == 'zh-classic' and 'zh-classical' in self.languages():
+ self.__code = 'zh-classical'
+ # database hack (database is varchar[10] -> zh-classical
+ # is cut to zh-classic.
+ else:
+ raise NoSuchSite("Language %s does not exist in family %s"
+ % (self.__code, self.__family.name))
+
+ self._username = [user, sysop]
+
+ # following are for use with lock_page and unlock_page methods
+ self._pagemutex = threading.Lock()
+ self._locked_pages = []
+
+ @property
+ def throttle(self):
+ """Return this Site's throttle. Initialize a new one if needed."""
+
+ if not hasattr(self, "_throttle"):
+ self._throttle = Throttle(self, multiplydelay=True,
+ verbosedelay=True)
+ try:
+ self.login(False)
+ except pywikibot.NoUsername:
+ pass
+ return self._throttle
+
+ @property
+ def family(self):
+ """The Family object for this Site's wiki family."""
+
+ return self.__family
+
+ @property
+ def code(self):
+ """The identifying code for this Site."""
+
+ return self.__code
+
+ @property
+ def lang(self):
+ """The ISO language code for this Site.
+
+ Presumed to be equal to the wiki prefix, but this can be overridden.
+
+ """
+ return self.__code
+
+ def __cmp__(self, other):
+ """Perform equality and inequality tests on Site objects."""
+
+ if not isinstance(other, BaseSite):
+ return 1
+ if self.family == other.family:
+ return cmp(self.code, other.code)
+ return cmp(self.family.name, other.family.name)
+
+ def user(self):
+ """Return the currently-logged in bot user, or None."""
+
+ if self.logged_in(True):
+ return self._username[True]
+ elif self.logged_in(False):
+ return self._username[False]
+ return None
+
+ def username(self, sysop = False):
+ return self._username[sysop]
+
+ def __getattr__(self, attr):
+ """Calls to methods not defined in this object are passed to Family."""
+
+ if hasattr(self.__class__, attr):
+ return self.__class__.attr
+ try:
+ method = getattr(self.family, attr)
+ f = lambda *args, **kwargs: \
+ method(self.code, *args, **kwargs)
+ if hasattr(method, "__doc__"):
+ f.__doc__ = method.__doc__
+ return f
+ except AttributeError:
+ raise AttributeError("%s instance has no attribute '%s'"
+ % (self.__class__.__name__, attr) )
+
+ def sitename(self):
+ """Return string representing this Site's name and language."""
+
+ return self.family.name+':'+self.code
+
+ __str__ = sitename
+
+ def __repr__(self):
+ return 'Site("%s", "%s")' % (self.code, self.family.name)
+
+ def __hash__(self):
+ return hash(repr(self))
+
+ def linktrail(self):
+ """Return regex for trailing chars displayed as part of a link.
+
+ Returns a string, not a compiled regular expression object.
+
+ This reads from the family file, and ''not'' from
+ [[MediaWiki:Linktrail]], because the MW software currently uses a
+ built-in linktrail from its message files and ignores the wiki
+ value.
+
+ """
+ return self.family.linktrail(self.code)
+
+ def languages(self):
+ """Return list of all valid language codes for this site's Family."""
+
+ return self.family.langs.keys()
+
+ def validLanguageLinks(self):
+ """Return list of language codes that can be used in interwiki links."""
+
+ nsnames = sum(self.namespaces().values(), [])
+ return [l for l in self.languages()
+ if l[:1].upper() + l[1:] not in self.namespaces()]
+
+ def ns_index(self, namespace):
+ """Given a namespace name, return its int index, or None if invalid."""
+
+ for ns in self.namespaces():
+ if namespace.lower() in [name.lower()
+ for name in self.namespaces()[ns]]:
+ return ns
+ return None
+
+ getNamespaceIndex = ns_index # for backwards-compatibility
+
+ def namespaces(self):
+ """Return dict of valid namespaces on this wiki."""
+
+ return self._namespaces
+
+ def ns_normalize(self, value):
+ """Return canonical local form of namespace name.
+
+ @param value: A namespace name
+ @type value: unicode
+
+ """
+ index = self.ns_index(value)
+ return self.namespace(index)
+
+ normalizeNamespace = ns_normalize # for backwards-compatibility
+
+ def redirect(self, default=True):
+ """Return the localized redirect tag for the site.
+
+ If default is True, falls back to 'REDIRECT' if the site has no
+ special redirect tag.
+
+ """
+ if default:
+ return self.family.redirect.get(self.code, [u"REDIRECT"])[0]
+ else:
+ return self.family.redirect.get(self.code, None)
+
+ def lock_page(self, page, block=True):
+ """Lock page for writing. Must be called before writing any page.
+
+ We don't want different threads trying to write to the same page
+ at the same time, even to different sections.
+
+ @param page: the page to be locked
+ @type page: pywikibot.Page
+ @param block: if true, wait until the page is available to be locked;
+ otherwise, raise an exception if page can't be locked
+
+ """
+ self._pagemutex.acquire()
+ try:
+ while page in self._locked_pages:
+ if not block:
+ raise PageInUse
+ time.sleep(.25)
+ self._locked_pages.append(page.title(withSection=False))
+ finally:
+ self._pagemutex.release()
+
+ def unlock_page(self, page):
+ """Unlock page. Call as soon as a write operation has completed.
+
+ @param page: the page to be locked
+ @type page: pywikibot.Page
+
+ """
+ self._pagemutex.acquire()
+ try:
+ self._locked_pages.remove(page.title(withSection=False))
+ finally:
+ self._pagemutex.release()
+
+ def disambcategory(self):
+ """Return Category in which disambig pages are listed."""
+
+ try:
+ name = self.namespace(14)+':'+self.family.disambcatname[self.code]
+ except KeyError:
+ raise Error(u"No disambiguation category name found for %(site)s"
+ % {'site': self})
+ return pywikibot.Category(pywikibot.Link(name, self))
+
+ def linkto(self, title, othersite = None):
+ """Return unicode string in the form of a wikilink to 'title'
+
+ Use optional Site argument 'othersite' to generate an interwiki link.
+
+ """
+ logger.debug("Site.linkto() method is deprecated; use pywikibot.Link")
+ return pywikibot.Link(title, self).astext(othersite)
+
+ def isInterwikiLink(self, s):
+ """Return True if s is in the form of an interwiki link.
+
+ If a link object constructed using "s" as the link text parses as
+ belonging to a different site, this method returns True.
+
+ """
+ return (pywikibot.Link(s, self).site != self)
+
+ def redirectRegex(self):
+ """Return a compiled regular expression matching on redirect pages.
+
+ Group 1 in the regex match object will be the target title.
+
+ """
+ #TODO: is this needed, since the API identifies redirects?
+ # (maybe, the API can give false positives)
+ default = 'REDIRECT'
+ try:
+ keywords = set(self.family.redirect[self.code])
+ keywords.add(default)
+ pattern = r'(?:' + '|'.join(keywords) + ')'
+ except KeyError:
+ # no localized keyword for redirects
+ pattern = r'%s' % default
+ # A redirect starts with hash (#), followed by a keyword, then
+ # arbitrary stuff, then a wikilink. The wikilink may contain
+ # a label, although this is not useful.
+ return re.compile(r'\s*#%(pattern)s\s*:?\s*\[\[(.+?)(?:\|.*?)?\]\]'
+ % locals(),
+ re.IGNORECASE | re.UNICODE | re.DOTALL)
+
+ # namespace shortcuts for backwards-compatibility
+
+ def special_namespace(self):
+ return self.namespace(-1)
+
+ def image_namespace(self):
+ return self.namespace(6)
+
+ def mediawiki_namespace(self):
+ return self.namespace(8)
+
+ def template_namespace(self):
+ return self.namespace(10)
+
+ def category_namespace(self):
+ return self.namespace(14)
+
+ def category_namespaces(self):
+ return self.namespace(14, all=True)
+
+ # site-specific formatting preferences
+
+ def category_on_one_line(self):
+ """Return True if this site wants all category links on one line."""
+
+ return self.code in self.family.category_on_one_line
+
+ def interwiki_putfirst(self):
+ """Return list of language codes for ordering of interwiki links."""
+
+ return self.family.interwiki_putfirst.get(self.code, None)
+
+ def interwiki_putfirst_doubled(self, list_of_links):
+ # TODO: is this even needed? No family in the framework has this
+ # dictionary defined!
+ if self.lang in self.family.interwiki_putfirst_doubled:
+ if len(list_of_links) >= \
+ self.family.interwiki_putfirst_doubled[self.lang][0]:
+ links2 = [lang.language() for lang in list_of_links]
+ result = []
+ for lang in self.family.interwiki_putfirst_doubled[self.lang][1]:
+ try:
+ result.append(list_of_links[links2.index(lang)])
+ except ValueError:
+ pass
+ return result
+ else:
+ return False
+ else:
+ return False
+
+ def getSite(self, code):
+ """Return Site object for language 'code' in this Family."""
+
+ return pywikibot.Site(code=code, fam=self.family, user=self.user)
+
+ # deprecated methods for backwards-compatibility
+
+ def fam(self):
+ """Return Family object for this Site."""
+ return self.family
+
+ def urlEncode(self, query):
+ """DEPRECATED"""
+ return urllib.urlencode(query)
+
+ def getUrl(self, path, retry=True, sysop=False, data=None,
+ compress=True, no_hostname=False, cookie_only=False):
+ """DEPRECATED.
+
+ Retained for compatibility only. All arguments except path and data
+ are ignored.
+
+ """
+ if data:
+ if not isinstance(data, basestring):
+ data = urllib.urlencode(data)
+ return pywikibot.comms.data.request(self, path, method="PUT",
+ body=data)
+ else:
+ return pywikibot.comms.data.request(self, path)
+
+ def postForm(self, address, predata, sysop=False, cookies=None):
+ """DEPRECATED"""
+ return self.getUrl(address, data=predata)
+
+ def postData(self, address, data, contentType=None, sysop=False,
+ compress=True, cookies=None):
+ """DEPRECATED"""
+ return self.getUrl(address, data=data)
+
+ # unsupported methods from version 1
+
+ def checkCharset(self, charset):
+ raise NotImplementedError
+ def getToken(self, getalways=True, getagain=False, sysop=False):
+ raise NotImplementedError
+ def export_address(self):
+ raise NotImplementedError
+ def move_address(self):
+ raise NotImplementedError
+ def delete_address(self, s):
+ raise NotImplementedError
+ def undelete_view_address(self, s, ts=''):
+ raise NotImplementedError
+ def undelete_address(self):
+ raise NotImplementedError
+ def protect_address(self, s):
+ raise NotImplementedError
+ def unprotect_address(self, s):
+ raise NotImplementedError
+ def put_address(self, s):
+ raise NotImplementedError
+ def get_address(self, s):
+ raise NotImplementedError
+ def nice_get_address(self, s):
+ raise NotImplementedError
+ def edit_address(self, s):
+ raise NotImplementedError
+ def purge_address(self, s):
+ raise NotImplementedError
+ def block_address(self):
+ raise NotImplementedError
+ def unblock_address(self):
+ raise NotImplementedError
+ def blocksearch_address(self, s):
+ raise NotImplementedError
+ def linksearch_address(self, s, limit=500, offset=0):
+ raise NotImplementedError
+ def search_address(self, q, n=50, ns=0):
+ raise NotImplementedError
+ def allpages_address(self, s, ns = 0):
+ raise NotImplementedError
+ def log_address(self, n=50, mode = ''):
+ raise NotImplementedError
+ def newpages_address(self, n=50):
+ raise NotImplementedError
+ def longpages_address(self, n=500):
+ raise NotImplementedError
+ def shortpages_address(self, n=500):
+ raise NotImplementedError
+ def unusedfiles_address(self, n=500):
+ raise NotImplementedError
+ def categories_address(self, n=500):
+ raise NotImplementedError
+ def deadendpages_address(self, n=500):
+ raise NotImplementedError
+ def ancientpages_address(self, n=500):
+ raise NotImplementedError
+ def lonelypages_address(self, n=500):
+ raise NotImplementedError
+ def protectedpages_address(self, n=500):
+ raise NotImplementedError
+ def unwatchedpages_address(self, n=500):
+ raise NotImplementedError
+ def uncategorizedcategories_address(self, n=500):
+ raise NotImplementedError
+ def uncategorizedimages_address(self, n=500):
+ raise NotImplementedError
+ def uncategorizedpages_address(self, n=500):
+ raise NotImplementedError
+ def unusedcategories_address(self, n=500):
+ raise NotImplementedError
+ def withoutinterwiki_address(self, n=500):
+ raise NotImplementedError
+ def references_address(self, s):
+ raise NotImplementedError
+ def allmessages_address(self):
+ raise NotImplementedError
+ def upload_address(self):
+ raise NotImplementedError
+ def double_redirects_address(self, default_limit = True):
+ raise NotImplementedError
+ def broken_redirects_address(self, default_limit = True):
+ raise NotImplementedError
+ def login_address(self):
+ raise NotImplementedError
+ def captcha_image_address(self, id):
+ raise NotImplementedError
+ def watchlist_address(self):
+ raise NotImplementedError
+ def contribs_address(self, target, limit=500, offset=''):
+ raise NotImplementedError
+
+
+class APISite(BaseSite):
+ """API interface to MediaWiki site.
+
+ Do not use directly; use pywikibot.Site function.
+
+ """
+## Site methods from version 1.0 (as these are implemented in this file,
+## or declared deprecated/obsolete, they will be removed from this list)
+##########
+## cookies: return user's cookies as a string
+##
+## urlEncode: Encode a query to be sent using an http POST request.
+## postForm: Post form data to an address at this site.
+## postData: Post encoded form data to an http address at this site.
+##
+## shared_image_repository: Return tuple of image repositories used by this
+## site.
+## version: Return MediaWiki version string from Family file.
+## versionnumber: Return int identifying the MediaWiki version.
+## live_version: Return version number read from Special:Version.
+## checkCharset(charset): Warn if charset doesn't match family file.
+##
+## linktrail: Return regex for trailing chars displayed as part of a link.
+## disambcategory: Category in which disambiguation pages are listed.
+##
+## Methods that yield Page objects derived from a wiki's Special: pages
+## (note, some methods yield other information in a tuple along with the
+## Pages; see method docs for details) --
+##
+## newpages(): Special:Newpages
+## newimages(): Special:Log&type=upload
+## longpages(): Special:Longpages
+## shortpages(): Special:Shortpages
+## deadendpages(): Special:Deadendpages
+## ancientpages(): Special:Ancientpages
+## lonelypages(): Special:Lonelypages
+## unwatchedpages(): Special:Unwatchedpages (sysop accounts only)
+## uncategorizedcategories(): Special:Uncategorizedcategories (yields
+## Category objects)
+## uncategorizedpages(): Special:Uncategorizedpages
+## uncategorizedimages(): Special:Uncategorizedimages (yields
+## ImagePage objects)
+## unusedcategories(): Special:Unusuedcategories (yields Category)
+## unusedfiles(): Special:Unusedimages (yields ImagePage)
+## withoutinterwiki: Special:Withoutinterwiki
+## linksearch: Special:Linksearch
+
+ def __init__(self, code, fam=None, user=None, sysop=None):
+ BaseSite.__init__(self, code, fam, user, sysop)
+ self._namespaces = {
+ # these are the MediaWiki built-in names, which always work
+ # localized names are loaded later upon accessing the wiki
+ # namespace prefixes are always case-insensitive, but the
+ # canonical forms are capitalized
+ -2: [u"Media"],
+ -1: [u"Special"],
+ 0: [u""],
+ 1: [u"Talk"],
+ 2: [u"User"],
+ 3: [u"User talk"],
+ 4: [u"Project"],
+ 5: [u"Project talk"],
+ 6: [u"Image"],
+ 7: [u"Image talk"],
+ 8: [u"MediaWiki"],
+ 9: [u"MediaWiki talk"],
+ 10: [u"Template"],
+ 11: [u"Template talk"],
+ 12: [u"Help"],
+ 13: [u"Help talk"],
+ 14: [u"Category"],
+ 15: [u"Category talk"],
+ }
+ self.sitelock = threading.Lock()
+ self._msgcache = {}
+ return
+
+# ANYTHING BELOW THIS POINT IS NOT YET IMPLEMENTED IN __init__()
+ self.nocapitalize = self.__code in self.family.nocapitalize
+ # Calculating valid languages took quite long, so we calculate it once
+ # in initialization instead of each time it is used.
+ self._validlanguages = []
+ for language in self.languages():
+ if not language[:1].upper() + language[1:] in self.namespaces():
+ self._validlanguages.append(language)
+
+ def logged_in(self, sysop=False):
+ """Return True if logged in with specified privileges, otherwise False.
+
+ @param sysop: if True, require sysop privileges.
+
+ """
+ if self.userinfo['name'] != self._username[sysop]:
+ return False
+ return (not sysop) or 'sysop' in self.userinfo['groups']
+
+ def loggedInAs(self, sysop = False):
+ """Return the current username if logged in, otherwise return None.
+
+ DEPRECATED (use .user() method instead)
+
+ """
+ logger.debug("Site.loggedInAs() method is deprecated.")
+ return self.logged_in(sysop) and self.user()
+
+ def login(self, sysop=False):
+ """Log the user in if not already logged in."""
+ if not hasattr(self, "_siteinfo"):
+ self._getsiteinfo()
+ # check whether a login cookie already exists for this user
+ if hasattr(self, "_userinfo"):
+ if self.userinfo['name'] == self._username[sysop]:
+ return
+ if not self.logged_in(sysop):
+ loginMan = api.LoginManager(site=self, sysop=sysop,
+ user=self._username[sysop])
+ if loginMan.login(retry = True):
+ self._username[sysop] = loginMan.username
+ if hasattr(self, "_userinfo"):
+ del self._userinfo
+ self.getuserinfo()
+
+ forceLogin = login # alias for backward-compatibility
+
+ def getuserinfo(self):
+ """Retrieve userinfo from site and store in _userinfo attribute.
+
+ self._userinfo will be a dict with the following keys and values:
+
+ - id: user id (numeric str)
+ - name: username (if user is logged in)
+ - anon: present if user is not logged in
+ - groups: list of groups (could be empty)
+ - rights: list of rights (could be empty)
+ - message: present if user has a new message on talk page
+ - blockinfo: present if user is blocked (dict)
+
+ """
+ if (not hasattr(self, "_userinfo")
+ or "rights" not in self._userinfo
+ or self._userinfo['name']
+ != self._username["sysop" in self._userinfo["groups"]]):
+ uirequest = api.Request(
+ site=self,
+ action="query",
+ meta="userinfo",
+ uiprop="blockinfo|hasmsg|groups|rights"
+ )
+ uidata = uirequest.submit()
+ assert 'query' in uidata, \
+ "API userinfo response lacks 'query' key"
+ assert 'userinfo' in uidata['query'], \
+ "API userinfo response lacks 'userinfo' key"
+ self._userinfo = uidata['query']['userinfo']
+ return self._userinfo
+
+ userinfo = property(fget=getuserinfo, doc=getuserinfo.__doc__)
+
+ def is_blocked(self, sysop=False):
+ """Return true if and only if user is blocked.
+
+ @param sysop: If true, log in to sysop account (if available)
+
+ """
+ if not self.logged_in(sysop):
+ self.login(sysop)
+ return 'blockinfo' in self._userinfo
+
+ def isBlocked(self, sysop=False):
+ """Deprecated synonym for is_blocked"""
+ logger.debug(
+ "Site method 'isBlocked' should be changed to 'is_blocked'")
+ return self.is_blocked(sysop)
+
+ def checkBlocks(self, sysop = False):
+ """Check if the user is blocked, and raise an exception if so."""
+ if self.is_blocked(sysop):
+ # User blocked
+ raise UserBlocked('User is blocked in site %s' % self)
+
+ def has_right(self, right, sysop=False):
+ """Return true if and only if the user has a specific right.
+
+ Possible values of 'right' may vary depending on wiki settings,
+ but will usually include:
+
+ * Actions: edit, move, delete, protect, upload
+ * User levels: autoconfirmed, sysop, bot
+
+ """
+ if not self.logged_in(sysop):
+ self.login(sysop)
+ return right.lower() in self._userinfo['rights']
+
+ def isAllowed(self, right, sysop=False):
+ """Deprecated; retained for backwards-compatibility"""
+ logger.debug("Site.isAllowed() method is deprecated; use has_right()")
+ return self.has_right(right, sysop)
+
+ def has_group(self, group, sysop=False):
+ """Return true if and only if the user is a member of specified group.
+
+ Possible values of 'group' may vary depending on wiki settings,
+ but will usually include bot.
+
+ """
+ if not self.logged_in(sysop):
+ self.login(sysop)
+ return group.lower() in self._userinfo['groups']
+
+ def messages(self, sysop=False):
+ """Returns true if the user has new messages, and false otherwise."""
+ if not self.logged_in(sysop):
+ self.login(sysop)
+ return 'hasmsg' in self._userinfo
+
+ def mediawiki_message(self, key):
+ """Return the MediaWiki message text for key "key" """
+ if not key in self._msgcache:
+ msg_query = api.QueryGenerator(site=self, meta="allmessages",
+ amfilter=key)
+ for msg in msg_query:
+ if msg['name'] == key and not 'missing' in msg:
+ self._msgcache[key] = msg['*']
+ break
+ else:
+ raise KeyError("Site %(self)s has no message '%(key)s'"
+ % locals())
+ return self._msgcache[key]
+
+ def has_mediawiki_message(self, key):
+ """Return True iff this site defines a MediaWiki message for 'key'."""
+ try:
+ v = self.mediawiki_message(key)
+ return True
+ except KeyError:
+ return False
+
+ def getcurrenttimestamp(self):
+ """Return (Mediawiki) timestamp, {{CURRENTTIMESTAMP}}, the server time.
+
+ Format is yyyymmddhhmmss
+
+ """
+ r = api.Request(site=self,
+ action="parse",
+ text="{{CURRENTTIMESTAMP}}")
+ result = r.submit()
+ return re.search('\d+', result['parse']['text']['*']).group()
+
+ def _getsiteinfo(self):
+ """Retrieve siteinfo and namespaces from site."""
+ sirequest = api.Request(
+ site=self,
+ action="query",
+ meta="siteinfo",
+ siprop="general|namespaces|namespacealiases"
+ )
+ try:
+ sidata = sirequest.submit()
+ except api.APIError:
+ # hack for older sites that don't support 1.12 properties
+ # probably should delete if we're not going to support pre-1.12
+ sirequest = api.Request(
+ site=self,
+ action="query",
+ meta="siteinfo",
+ siprop="general|namespaces"
+ )
+ sidata = sirequest.submit()
+
+ assert 'query' in sidata, \
+ "API siteinfo response lacks 'query' key"
+ sidata = sidata['query']
+ assert 'general' in sidata, \
+ "API siteinfo response lacks 'general' key"
+ assert 'namespaces' in sidata, \
+ "API siteinfo response lacks 'namespaces' key"
+ self._siteinfo = sidata['general']
+ nsdata = sidata['namespaces']
+ for nskey in nsdata:
+ ns = int(nskey)
+ if ns in self._namespaces:
+ if nsdata[nskey]["*"] in self._namespaces[ns]:
+ continue
+ # this is the preferred form so it goes at front of list
+ self._namespaces[ns].insert(0, nsdata[nskey]["*"])
+ else:
+ self._namespaces[ns] = [nsdata[nskey]["*"]]
+ if 'namespacealiases' in sidata:
+ aliasdata = sidata['namespacealiases']
+ for item in aliasdata:
+ if item["*"] in self._namespaces[int(item['id'])]:
+ continue
+ # this is a less preferred form so it goes at the end
+ self._namespaces[int(item['id'])].append(item["*"])
+
+ @property
+ def siteinfo(self):
+ """Site information dict."""
+
+ if not hasattr(self, "_siteinfo"):
+ self._getsiteinfo()
+ return self._siteinfo
+
+ def case(self):
+ """Return this site's capitalization rule."""
+
+ return self.siteinfo['case']
+
+ def language(self):
+ """Return the code for the language of this Site."""
+
+ return self.siteinfo['lang']
+
+ lang = property(fget=language, doc=language.__doc__)
+
+ def namespaces(self):
+ """Return dict of valid namespaces on this wiki."""
+
+ if not hasattr(self, "_siteinfo"):
+ self._getsiteinfo()
+ return self._namespaces
+
+ def namespace(self, num, all=False):
+ """Return string containing local name of namespace 'num'.
+
+ If optional argument 'all' is true, return a list of all recognized
+ values for this namespace.
+
+ """
+ if all:
+ return self.namespaces()[num]
+ return self.namespaces()[num][0]
+
+ def live_version(self):
+ """Return the 'real' version number found on [[Special:Version]]
+
+ Return value is a tuple (int, int, str) of the major and minor
+ version numbers and any other text contained in the version.
+
+ """
+ versionstring = self.siteinfo['generator']
+ m = re.match(r"^MediaWiki ([0-9]+)\.([0-9]+)(.*)$", versionstring)
+ if m:
+ return (int(m.group(1)), int(m.group(2)), m.group(3))
+ else:
+ return None
+
+ def loadpageinfo(self, page):
+ """Load page info from api and save in page attributes"""
+ title = page.title(withSection=False)
+ query = api.PropertyGenerator("info", site=self,
+ titles=title.encode(self.encoding()),
+ inprop="protection")
+ for pageitem in query:
+ if pageitem['title'] != title:
+ raise Error(
+ u"loadpageinfo: Query on %s returned data on '%s'"
+ % (page, pageitem['title']))
+ api.update_page(page, pageitem)
+
+ def loadimageinfo(self, page, history=False):
+ """Load image info from api and save in page attributes
+
+ @param history: if true, return the image's version history
+
+ """
+ title = page.title(withSection=False)
+ query = api.PropertyGenerator("imageinfo", site=self,
+ titles=title.encode(self.encoding()),
+ iiprop=["timestamp", "user", "comment",
+ "url", "size", "sha1", "mime",
+ "metadata", "archivename"])
+ if history:
+ query.request["iilimit"] = "max"
+ for pageitem in query:
+ if pageitem['title'] != title:
+ raise Error(
+ u"loadpageinfo: Query on %s returned data on '%s'"
+ % (page, pageitem['title']))
+ api.update_page(page, pageitem)
+ if history:
+ return pageitem['imageinfo']
+
+ def page_exists(self, page):
+ """Return True if and only if page is an existing page on site."""
+ if not hasattr(page, "_pageid"):
+ self.loadpageinfo(page)
+ return page._pageid > 0
+
+ def page_restrictions(self, page):
+ """Returns a dictionary reflecting page protections"""
+ if not self.page_exists(page):
+ raise NoPage(u'No page %s.' % page)
+ if not hasattr(page, "_protection"):
+ self.loadpageinfo(page)
+ return page._protection
+
+ def page_can_be_edited(self, page):
+ """
+ Returns True if and only if:
+ - page is unprotected, and bot has an account for this site, or
+ - page is protected, and bot has a sysop account for this site.
+
+ """
+ rest = self.page_restrictions(page)
+ sysop_protected = rest.has_key('edit') and rest['edit'][0] == 'sysop'
+ try:
+ api.LoginManager(site=self, sysop=sysop_protected)
+ except NoUsername:
+ return False
+ return True
+
+ def page_isredirect(self, page):
+ """Return True if and only if page is a redirect."""
+ if not hasattr(page, "_redir"):
+ self.loadpageinfo(page)
+ return bool(page._redir)
+
+ def getredirtarget(self, page):
+ """Return Page object for the redirect target of page."""
+ if not hasattr(page, "_redir"):
+ self.loadpageinfo(page)
+ if not page._redir:
+ raise pywikibot.IsNotRedirectPage(page.title())
+ title = page.title(withSection=False)
+ query = api.Request(site=self, action="query", property="info",
+ inprop="protection|talkid|subjectid",
+ titles=title.encode(self.encoding()),
+ redirects="")
+ result = query.submit()
+ if "query" not in result or "redirects" not in result["query"]:
+ raise RuntimeError(
+ "getredirtarget: No 'redirects' found for page %s."
+ % title)
+ redirmap = dict((item['from'], item['to'])
+ for item in result['query']['redirects'])
+ if title not in redirmap:
+ raise RuntimeError(
+ "getredirtarget: 'redirects' contains no key for page %s."
+ % title)
+ if "pages" not in result['query']:
+ # no "pages" element indicates a circular redirect
+ raise pywikibot.CircularRedirect(redirmap[title])
+ for pagedata in result['query']['pages'].values():
+ # there should be only one value in 'pages', and it is the target
+ if pagedata['title'] not in redirmap.values():
+ raise RuntimeError(
+ "getredirtarget: target page '%s' not found in 'redirects'"
+ % pagedata['title'])
+ target = pywikibot.Page(self, pagedata['title'], pagedata['ns'])
+ api.update_page(target, pagedata)
+ page._redir = target
+
+ def preloadpages(self, pagelist, groupsize=60):
+ """Return a generator to a list of preloaded pages.
+
+ Note that [at least in current implementation] pages may be iterated
+ in a different order than in the underlying pagelist.
+
+ @param pagelist: an iterable that returns Page objects
+ @param groupsize: how many Pages to query at a time
+ @type groupsize: int
+
+ """
+ from pywikibot.tools import itergroup
+ for sublist in itergroup(pagelist, groupsize):
+ pageids = [str(p._pageid) for p in sublist
+ if hasattr(p, "_pageid")
+ and p._pageid > 0]
+ cache = dict((p.title(withSection=False), p) for p in sublist)
+ rvgen = api.PropertyGenerator("revisions|info", site=self)
+ rvgen.limit = -1
+ if len(pageids) == len(sublist):
+ # only use pageids if all pages have them
+ rvgen.request["pageids"] = "|".join(pageids)
+ else:
+ rvgen.request["titles"] = "|".join(cache.keys())
+ rvgen.request[u"rvprop"] = \
+ u"ids|flags|timestamp|user|comment|content"
+ logger.info(u"Retrieving %s pages from %s."
+ % (len(cache), self)
+ )
+ for pagedata in rvgen:
+ logger.debug("Preloading %s" % pagedata)
+ try:
+ if pagedata['title'] not in cache:
+ raise Error(
+ u"preloadpages: Query returned unexpected title '%s'"
+ % pagedata['title']
+ )
+ except KeyError:
+ logger.debug("No 'title' in %s" % pagedata)
+ logger.debug("pageids=%s" % pageids)
+ logger.debug("titles=%s" % cache.keys())
+ continue
+ page = cache[pagedata['title']]
+ api.update_page(page, pagedata)
+ yield page
+
+ def token(self, page, tokentype):
+ """Return token retrieved from wiki to allow changing page content.
+
+ @param page: the Page for which a token should be retrieved
+ @param tokentype: the type of token (e.g., "edit", "move", "delete");
+ see API documentation for full list of types
+
+ """
+ query = api.PropertyGenerator("info|revisions", site=self,
+ titles=page.title(withSection=False),
+ intoken=tokentype)
+ for item in query:
+ if item['title'] != page.title(withSection=False):
+ raise Error(
+ u"token: Query on page %s returned data on page [[%s]]"
+ % (page.title(withSection=False, asLink=True),
+ item['title']))
+ api.update_page(page, item)
+ logging.debug(str(item))
+ return item[tokentype + "token"]
+
+ # following group of methods map more-or-less directly to API queries
+
+ def pagebacklinks(self, page, followRedirects=False, filterRedirects=None,
+ namespaces=None):
+ """Iterate all pages that link to the given page.
+
+ @param page: The Page to get links to.
+ @param followRedirects: Also return links to redirects pointing to
+ the given page.
+ @param filterRedirects: If True, only return redirects to the given
+ page. If False, only return non-redirect links. If None, return
+ both (no filtering).
+ @param namespaces: If present, only return links from the namespaces
+ in this list.
+
+ """
+ bltitle = page.title(withSection=False).encode(self.encoding())
+ blgen = api.PageGenerator("backlinks", gbltitle=bltitle, site=self)
+ if isinstance(namespaces, list):
+ blgen.request["gblnamespace"] = u"|".join(unicode(ns)
+ for ns in namespaces)
+ elif namespaces is not None:
+ blgen.request["gblnamespace"] = str(namespaces)
+ if filterRedirects is not None:
+ blgen.request["gblfilterredir"] = filterRedirects and "redirects"\
+ or "nonredirects"
+ if followRedirects:
+ # bug: see http://bugzilla.wikimedia.org/show_bug.cgi?id=16218
+ # links identified by MediaWiki as redirects may not really be,
+ # so we have to check each "redirect" page and see if it
+ # really redirects to this page
+ blgen.request["gblfilterredir"] = "nonredirects"
+ redirgen = api.PageGenerator("backlinks", gbltitle=bltitle,
+ site=self, gblfilterredir="redirects")
+ if "gblnamespace" in blgen.request:
+ redirgen.request["gblnamespace"] = blgen.request["gblnamespace"]
+ genlist = [blgen]
+ for redir in redirgen:
+ if redir.getRedirectTarget() == page:
+ genlist.append(
+ self.pagebacklinks(
+ redir, True, None, namespaces))
+ import itertools
+ return itertools.chain(*genlist)
+ return blgen
+
+ def page_embeddedin(self, page, filterRedirects=None, namespaces=None):
+ """Iterate all pages that embedded the given page as a template.
+
+ @param page: The Page to get inclusions for.
+ @param filterRedirects: If True, only return redirects that embed
+ the given page. If False, only return non-redirect links. If
+ None, return both (no filtering).
+ @param namespaces: If present, only return links from the namespaces
+ in this list.
+
+ """
+ eititle = page.title(withSection=False).encode(self.encoding())
+ eigen = api.PageGenerator("embeddedin", geititle=eititle, site=self)
+ if isinstance(namespaces, list):
+ eigen.request["geinamespace"] = u"|".join(unicode(ns)
+ for ns in namespaces)
+ elif namespaces is not None:
+ eigen.request["geinamespace"] = str(namespaces)
+ if filterRedirects is not None:
+ eigen.request["geifilterredir"] = filterRedirects and "redirects"\
+ or "nonredirects"
+ return eigen
+
+ def pagereferences(self, page, followRedirects=False, filterRedirects=None,
+ withTemplateInclusion=True, onlyTemplateInclusion=False,
+ namespaces=None):
+ """Convenience method combining pagebacklinks and page_embeddedin."""
+
+ if onlyTemplateInclusion:
+ return self.page_embeddedin(page, namespaces=namespaces)
+ if not withTemplateInclusion:
+ return self.pagebacklinks(page, followRedirects,
+ namespaces=namespaces)
+ import itertools
+ return itertools.chain(
+ self.pagebacklinks(page, followRedirects,
+ filterRedirects, namespaces=namespaces),
+ self.page_embeddedin(page, filterRedirects,
+ namespaces=namespaces)
+ )
+
+ def pagelinks(self, page, namespaces=None, follow_redirects=False,
+ limit=None):
+ """Iterate internal wikilinks contained (or transcluded) on page.
+
+ @param namespaces: Only iterate pages in these namespaces (default: all)
+ @type namespaces: list of ints
+ @param follow_redirects: if True, yields the target of any redirects,
+ rather than the redirect page
+
+ """
+ plgen = api.PageGenerator("links", site=self)
+ if isinstance(limit, int):
+ plgen.limit = limit
+ if hasattr(page, "_pageid"):
+ plgen.request['pageids'] = str(page._pageid)
+ else:
+ pltitle = page.title(withSection=False).encode(self.encoding())
+ plgen.request['titles'] = pltitle
+ if follow_redirects:
+ plgen.request['redirects'] = ''
+ if isinstance(namespaces, list):
+ plgen.request["gplnamespace"] = u"|".join(unicode(ns)
+ for ns in namespaces)
+ elif namespaces is not None:
+ plgen.request["gplnamespace"] = str(namespaces)
+ return plgen
+
+ @deprecate_arg("withSortKey", None) # Sortkey doesn't work with generator
+ def pagecategories(self, page, withSortKey=None):
+ """Iterate categories to which page belongs."""
+
+ clgen = api.CategoryPageGenerator("categories", site=self)
+ if hasattr(page, "_pageid"):
+ clgen.request['pageids'] = str(page._pageid)
+ else:
+ cltitle = page.title(withSection=False).encode(self.encoding())
+ clgen.request['titles'] = cltitle
+ return clgen
+
+ def pageimages(self, page):
+ """Iterate images used (not just linked) on the page."""
+
+ imtitle = page.title(withSection=False).encode(self.encoding())
+ imgen = api.ImagePageGenerator("images", titles=imtitle, site=self)
+ return imgen
+
+ def pagetemplates(self, page, namespaces=None):
+ """Iterate templates transcluded (not just linked) on the page."""
+
+ tltitle = page.title(withSection=False).encode(self.encoding())
+ tlgen = api.PageGenerator("templates", titles=tltitle, site=self)
+ if isinstance(namespaces, list):
+ tlgen.request["gtlnamespace"] = u"|".join(unicode(ns)
+ for ns in namespaces)
+ elif namespaces is not None:
+ tlgen.request["gtlnamespace"] = str(namespaces)
+ return tlgen
+
+ def categorymembers(self, category, namespaces=None, limit=None):
+ """Iterate members of specified category.
+
+ @param category: The Category to iterate.
+ @param namespaces: If present, only return category members from
+ these namespaces. For example, use namespaces=[14] to yield
+ subcategories, use namespaces=[6] to yield image files, etc. Note,
+ however, that the iterated values are always Page objects, even
+ if in the Category or Image namespace.
+ @type namespaces: list of ints
+ @param limit: maximum number of pages to iterate (default: all)
+ @type limit: int
+
+ """
+ if category.namespace() != 14:
+ raise Error(
+ u"categorymembers: non-Category page '%s' specified"
+ % category.title())
+ cmtitle = category.title(withSection=False).encode(self.encoding())
+ cmgen = api.PageGenerator("categorymembers", gcmtitle=cmtitle,
+ gcmprop="ids|title|sortkey", site=self)
+ if isinstance(namespaces, list):
+ cmgen.request["gcmnamespace"] = u"|".join(unicode(ns)
+ for ns in namespaces)
+ elif namespaces is not None:
+ cmgen.request["gcmnamespace"] = str(namespaces)
+ if isinstance(limit, int):
+ cmgen.limit = limit
+ return cmgen
+
+ def loadrevisions(self, page=None, getText=False, revids=None,
+ limit=None, startid=None, endid=None, starttime=None,
+ endtime=None, rvdir=None, user=None, excludeuser=None,
+ section=None, sysop=False):
+ """Retrieve and store revision information.
+
+ By default, retrieves the last (current) revision of the page,
+ I{unless} any of the optional parameters revids, startid, endid,
+ starttime, endtime, rvdir, user, excludeuser, or limit are
+ specified. Unless noted below, all parameters not specified
+ default to False.
+
+ If rvdir is False or not specified, startid must be greater than
+ endid if both are specified; likewise, starttime must be greater
+ than endtime. If rvdir is True, these relationships are reversed.
+
+ @param page: retrieve revisions of this Page (required unless ids
+ is specified)
+ @param getText: if True, retrieve the wiki-text of each revision;
+ otherwise, only retrieve the revision metadata (default)
+ @param section: if specified, retrieve only this section of the text
+ (getText must be True); section must be given by number (top of
+ the article is section 0), not name
+ @type section: int
+ @param revids: retrieve only the specified revision ids (required
+ unless page is specified)
+ @type revids: list of ints
+ @param limit: Retrieve no more than this number of revisions
+ @type limit: int
+ @param startid: retrieve revisions starting with this revid
+ @param endid: stop upon retrieving this revid
+ @param starttime: retrieve revisions starting at this timestamp
+ @param endtime: stop upon reaching this timestamp
+ @param rvdir: if false, retrieve newest revisions first (default);
+ if true, retrieve earliest first
+ @param user: retrieve only revisions authored by this user
+ @param excludeuser: retrieve all revisions not authored by this user
+ @param sysop: if True, switch to sysop account (if available) to
+ retrieve this page
+
+ """
+ latest = (revids is None and
+ startid is None and
+ endid is None and
+ starttime is None and
+ endtime is None and
+ rvdir is None and
+ user is None and
+ excludeuser is None and
+ limit is None) # if True, we are retrieving current revision
+
+ # check for invalid argument combinations
+ if page is None and revids is None:
+ raise ValueError(
+ "loadrevisions: either page or revids argument required")
+ if (startid is not None or endid is not None) and \
+ (starttime is not None or endtime is not None):
+ raise ValueError(
+ "loadrevisions: startid/endid combined with starttime/endtime")
+ if starttime is not None and endtime is not None:
+ if rvdir and starttime >= endtime:
+ raise ValueError(
+ "loadrevisions: starttime > endtime with rvdir=True")
+ if (not rvdir) and endtime >= starttime:
+ raise ValueError(
+ "loadrevisions: endtime > starttime with rvdir=False")
+ if startid is not None and endid is not None:
+ if rvdir and startid >= endid:
+ raise ValueError(
+ "loadrevisions: startid > endid with rvdir=True")
+ if (not rvdir) and endid >= startid:
+ raise ValueError(
+ "loadrevisions: endid > startid with rvdir=False")
+
+ # assemble API request
+ if revids is None:
+ rvtitle = page.title(withSection=False).encode(self.encoding())
+ rvgen = api.PropertyGenerator(u"info|revisions", titles=rvtitle,
+ site=self)
+ else:
+ if isinstance(revids, (int, basestring)):
+ ids = unicode(revids)
+ else:
+ ids = u"|".join(unicode(r) for r in revids)
+ rvgen = api.PropertyGenerator(u"info|revisions", revids=ids,
+ site=self)
+ if getText:
+ rvgen.request[u"rvprop"] = \
+ u"ids|flags|timestamp|user|comment|content"
+ if section is not None:
+ rvgen.request[u"rvsection"] = unicode(section)
+ if latest or "revids" in rvgen.request:
+ rvgen.limit = -1 # suppress use of rvlimit parameter
+ elif isinstance(limit, int):
+ rvgen.limit = limit
+ if rvdir:
+ rvgen.request[u"rvdir"] = u"newer"
+ elif rvdir is not None:
+ rvgen.request[u"rvdir"] = u"older"
+ if startid:
+ rvgen.request[u"rvstartid"] = startid
+ if endid:
+ rvgen.request[u"rvendid"] = endid
+ if starttime:
+ rvgen.request[u"rvstart"] = starttime
+ if endtime:
+ rvgen.request[u"rvend"] = endtime
+ if user:
+ rvgen.request[u"rvuser"] = user
+ elif excludeuser:
+ rvgen.request[u"rvexcludeuser"] = excludeuser
+ # TODO if sysop: something
+ rvgen.continuekey = "revisions"
+ for pagedata in rvgen:
+ if page is not None:
+ if pagedata['title'] != page.title(withSection=False):
+ raise Error(
+ u"loadrevisions: Query on %s returned data on '%s'"
+ % (page, pagedata['title']))
+ if pagedata.has_key('missing'):
+ raise NoPage(u'Page %s does not exist'
+ % page.title(asLink=True))
+ else:
+ page = Page(self, pagedata['title'])
+ api.update_page(page, pagedata)
+
+ def pageinterwiki(self, page):
+ # No such function in the API (this method isn't called anywhere)
+ raise NotImplementedError
+
+ def pagelanglinks(self, page):
+ """Iterate all interlanguage links on page, yielding Link objects."""
+ lltitle = page.title(withSection=False)
+ llquery = api.PropertyGenerator("langlinks",
+ titles=lltitle.encode(self.encoding()),
+ site=self)
+ for pageitem in llquery:
+ if pageitem['title'] != lltitle:
+ raise Error(
+ u"getlanglinks: Query on %s returned data on '%s'"
+ % (page, pageitem['title']))
+ if 'langlinks' not in pageitem:
+ continue
+ for linkdata in pageitem['langlinks']:
+ yield pywikibot.Link(linkdata['*'],
+ source=pywikibot.Site(linkdata['lang']))
+
+ def page_extlinks(self, page):
+ """Iterate all external links on page, yielding URL strings."""
+ eltitle = page.title(withSection=False)
+ elquery = api.PropertyGenerator("extlinks",
+ titles=eltitle.encode(self.encoding()),
+ site=self)
+ for pageitem in elquery:
+ if pageitem['title'] != eltitle:
+ raise RuntimeError(
+ "getlanglinks: Query on %s returned data on '%s'"
+ % (page, pageitem['title']))
+ if 'extlinks' not in pageitem:
+ continue
+ for linkdata in pageitem['extlinks']:
+ yield linkdata['*']
+
+ @deprecate_arg("throttle", None)
+ @deprecate_arg("includeredirects", "filterredir")
+ def allpages(self, start="!", prefix="", namespace=0, filterredir=None,
+ filterlanglinks=None, minsize=None, maxsize=None,
+ protect_type=None, protect_level=None, limit=None,
+ reverse=False, includeredirects=None):
+ """Iterate pages in a single namespace.
+
+ Note: parameters includeRedirects and throttle are deprecated and
+ included only for backwards compatibility.
+
+ @param start: Start at this title (page need not exist).
+ @param prefix: Only yield pages starting with this string.
+ @param namespace: Iterate pages from this (single) namespace
+ (default: 0)
+ @param filterredir: if True, only yield redirects; if False (and not
+ None), only yield non-redirects (default: yield both)
+ @param filterlanglinks: if True, only yield pages with language links;
+ if False (and not None), only yield pages without language links
+ (default: yield both)
+ @param minsize: if present, only yield pages at least this many
+ bytes in size
+ @param maxsize: if present, only yield pages at most this many bytes
+ in size
+ @param protect_type: only yield pages that have a protection of the
+ specified type
+ @type protect_type: str
+ @param protect_level: only yield pages that have protection at this
+ level; can only be used if protect_type is specified
+ @param limit: maximum number of pages to iterate (default: iterate
+ all pages in namespace)
+ @param reverse: if True, iterate in reverse Unicode lexigraphic
+ order (default: iterate in forward order)
+ @param includeredirects: DEPRECATED, use filterredirs instead
+
+ """
+ if not isinstance(namespace, int):
+ raise Error("allpages: only one namespace permitted.")
+ if includeredirects is not None:
+ logger.debug(
+"allpages: 'includeRedirects' argument is deprecated; use 'filterredirs'.")
+ if includeredirects:
+ if includeredirects == "only":
+ filterredirs = True
+ else:
+ filterredirs = None
+ else:
+ filterredirs = False
+
+ apgen = api.PageGenerator("allpages", gapnamespace=str(namespace),
+ gapfrom=start, site=self)
+ if prefix:
+ apgen.request["gapprefix"] = prefix
+ if filterredir is not None:
+ apgen.request["gapfilterredir"] = (filterredir
+ and "redirects"
+ or "nonredirects")
+ if filterlanglinks is not None:
+ apgen.request["gapfilterlanglinks"] = (filterlanglinks
+ and "withlanglinks"
+ or "withoutlanglinks")
+ if isinstance(minsize, int):
+ apgen.request["gapminsize"] = str(minsize)
+ if isinstance(maxsize, int):
+ apgen.request["gapmaxsize"] = str(maxsize)
+ if isinstance(protect_type, basestring):
+ apgen.request["gapprtype"] = protect_type
+ if isinstance(protect_level, basestring):
+ apgen.request["gapprlevel"] = protect_level
+ if isinstance(limit, int):
+ apgen.limit = limit
+ if reverse:
+ apgen.request["gapdir"] = "descending"
+ return apgen
+
+ def prefixindex(self, prefix, namespace=0, includeredirects=True):
+ """Yield all pages with a given prefix. Deprecated.
+
+ Use allpages() with the prefix= parameter instead of this method.
+
+ """
+ logger.debug("Site.prefixindex() is deprecated; use allpages instead.")
+ return self.allpages(prefix=prefix, namespace=namespace,
+ includeredirects=includeredirects)
+
+
+ def alllinks(self, start="!", prefix="", namespace=0, unique=False,
+ limit=None, fromids=False):
+ """Iterate all links to pages (which need not exist) in one namespace.
+
+ Note that, in practice, links that were found on pages that have
+ been deleted may not have been removed from the links table, so this
+ method can return false positives.
+
+ @param start: Start at this title (page need not exist).
+ @param prefix: Only yield pages starting with this string.
+ @param namespace: Iterate pages from this (single) namespace
+ (default: 0)
+ @param unique: If True, only iterate each link title once (default:
+ iterate once for each linking page)
+ @param limit: maximum number of pages to iterate (default: iterate
+ all pages in namespace)
+ @param fromids: if True, include the pageid of the page containing
+ each link (default: False) as the '_fromid' attribute of the Page;
+ cannot be combined with unique
+
+ """
+ if unique and fromids:
+ raise Error("alllinks: unique and fromids cannot both be True.")
+ if not isinstance(namespace, int):
+ raise Error("alllinks: only one namespace permitted.")
+ algen = api.ListGenerator("alllinks", alnamespace=str(namespace),
+ alfrom=start, site=self)
+ if prefix:
+ algen.request["alprefix"] = prefix
+ if isinstance(limit, int):
+ algen.limit = limit
+ if unique:
+ algen.request["alunique"] = ""
+ if fromids:
+ algen.request["alprop"] = "title|ids"
+ for link in algen:
+ p = pywikibot.Page(self, link['title'], link['ns'])
+ if fromids:
+ p._fromid = link['fromid']
+ yield p
+
+ def allcategories(self, start="!", prefix="", limit=None,
+ reverse=False):
+ """Iterate categories used (which need not have a Category page).
+
+ Iterator yields Category objects. Note that, in practice, links that
+ were found on pages that have been deleted may not have been removed
+ from the database table, so this method can return false positives.
+
+ @param start: Start at this category title (category need not exist).
+ @param prefix: Only yield categories starting with this string.
+ @param limit: maximum number of categories to iterate (default:
+ iterate all)
+ @param reverse: if True, iterate in reverse Unicode lexigraphic
+ order (default: iterate in forward order)
+
+ """
+ acgen = api.CategoryPageGenerator("allcategories",
+ gacfrom=start, site=self)
+ if prefix:
+ acgen.request["gacprefix"] = prefix
+ if isinstance(limit, int):
+ acgen.limit = limit
+ if reverse:
+ acgen.request["gacdir"] = "descending"
+ return acgen
+
+ def categories(self, number=10, repeat=False):
+ """Deprecated; retained for backwards-compatibility"""
+ logger.debug(
+ "Site.categories() method is deprecated; use .allcategories()")
+ if repeat:
+ limit = None
+ else:
+ limit = number
+ return self.allcategories(limit=limit)
+
+ def allusers(self, start="!", prefix="", limit=None, group=None):
+ """Iterate registered users, ordered by username.
+
+ Iterated values are dicts containing 'name', 'editcount',
+ 'registration', and (sometimes) 'groups' keys. 'groups' will be
+ present only if the user is a member of at least 1 group, and will
+ be a list of unicodes; all the other values are unicodes and should
+ always be present.
+
+ @param start: start at this username (name need not exist)
+ @param prefix: only iterate usernames starting with this substring
+ @param limit: maximum number of users to iterate (default: all)
+ @param group: only iterate users that are members of this group
+ @type group: str
+
+ """
+ augen = api.ListGenerator("allusers", aufrom=start,
+ auprop="editcount|groups|registration",
+ site=self)
+ if prefix:
+ augen.request["auprefix"] = prefix
+ if group:
+ augen.request["augroup"] = group
+ if isinstance(limit, int):
+ augen.limit = limit
+ return augen
+
+ def allimages(self, start="!", prefix="", minsize=None, maxsize=None,
+ limit=None, reverse=False, sha1=None, sha1base36=None):
+ """Iterate all images, ordered by image title.
+
+ Yields ImagePages, but these pages need not exist on the wiki.
+
+ @param start: start at this title (name need not exist)
+ @param prefix: only iterate titles starting with this substring
+ @param limit: maximum number of titles to iterate (default: all)
+ @param minsize: only iterate images of at least this many bytes
+ @param maxsize: only iterate images of no more than this many bytes
+ @param reverse: if True, iterate in reverse lexigraphic order
+ @param sha1: only iterate image (it is theoretically possible there
+ could be more than one) with this sha1 hash
+ @param sha1base36: same as sha1 but in base 36
+
+ """
+ aigen = api.ImagePageGenerator("allimages", gaifrom=start,
+ site=self)
+ if prefix:
+ aigen.request["gaiprefix"] = prefix
+ if isinstance(limit, int):
+ aigen.limit = limit
+ if isinstance(minsize, int):
+ aigen.request["gaiminsize"] = str(minsize)
+ if isinstance(maxsize, int):
+ aigen.request["gaimaxsize"] = str(maxsize)
+ if reverse:
+ aigen.request["gaidir"] = "descending"
+ if sha1:
+ aigen.request["gaisha1"] = sha1
+ if sha1base36:
+ aigen.request["gaisha1base36"] = sha1base36
+ return aigen
+
+ def blocks(self, starttime=None, endtime=None, reverse=False,
+ blockids=None, users=None, limit=None):
+ """Iterate all current blocks, in order of creation.
+
+ Note that logevents only logs user blocks, while this method
+ iterates all blocks including IP ranges. The iterator yields dicts
+ containing keys corresponding to the block properties (see
+ http://www.mediawiki.org/wiki/API:Query_-_Lists for documentation).
+
+ @param starttime: start iterating at this timestamp
+ @param endtime: stop iterating at this timestamp
+ @param reverse: if True, iterate oldest blocks first (default: newest)
+ @param blockids: only iterate blocks with these id numbers
+ @param users: only iterate blocks affecting these usernames or IPs
+ @param limit: maximum number of blocks to iterate (default: all)
+
+ """
+ if starttime and endtime:
+ if reverse:
+ if starttime > endtime:
+ raise pywikibot.Error(
+ "blocks: starttime must be before endtime with reverse=True")
+ else:
+ if endtime > starttime:
+ raise pywikibot.Error(
+ "blocks: endtime must be before starttime with reverse=False")
+ bkgen = api.ListGenerator("blocks", site=self)
+ bkgen.request["bkprop"] = \
+ "id|user|by|timestamp|expiry|reason|range|flags"
+ if starttime:
+ bkgen.request["bkstart"] = starttime
+ if endtime:
+ bkgen.request["bkend"] = endtime
+ if reverse:
+ bkgen.request["bkdir"] = "newer"
+ if blockids:
+ bkgen.request["bkids"] = blockids
+ if users:
+ bkgen.request["bkusers"] = users
+ if isinstance(limit, int):
+ bkgen.limit = limit
+ return bkgen
+
+ def exturlusage(self, url, protocol="http", namespaces=None,
+ limit=None):
+ """Iterate Pages that contain links to the given URL.
+
+ @param url: The URL to search for (without the protocol prefix);
+ this many include a '*' as a wildcard, only at the start of the
+ hostname
+ @param protocol: The protocol prefix (default: "http")
+ @param namespaces: Only iterate pages in these namespaces (default: all)
+ @type namespaces: list of ints
+ @param limit: Only iterate this many linking pages (default: all)
+
+ """
+ eugen = api.PageGenerator("exturlusage", geuquery=url,
+ geuprotocol=protocol, site=self)
+ if isinstance(namespaces, list):
+ eugen.request["geunamespace"] = u"|".join(unicode(ns)
+ for ns in namespaces)
+ elif namespaces is not None:
+ eugen.request["geunamespace"] = str(namespaces)
+ if isinstance(limit, int):
+ eugen.limit = limit
+ return eugen
+
+ def imageusage(self, image, namespaces=None, filterredir=None,
+ limit=None):
+ """Iterate Pages that contain links to the given ImagePage.
+
+ @param image: the image to search for (ImagePage need not exist on the wiki)
+ @type image: ImagePage
+ @param namespaces: Only iterate pages in these namespaces (default: all)
+ @type namespaces: list of ints
+ @param filterredir: if True, only yield redirects; if False (and not
+ None), only yield non-redirects (default: yield both)
+ @param limit: Only iterate this many linking pages (default: all)
+
+ """
+ iugen = api.PageGenerator("imageusage", site=self,
+ giutitle=image.title(withSection=False))
+ if isinstance(namespaces, list):
+ iugen.request["giunamespace"] = u"|".join(unicode(ns)
+ for ns in namespaces)
+ elif namespaces is not None:
+ iugen.request["giunamespace"] = str(namespaces)
+ if isinstance(limit, int):
+ iugen.limit = limit
+ if filterredir is not None:
+ iugen.request["giufilterredir"] = (filterredir and "redirects"
+ or "nonredirects")
+ return iugen
+
+ def logevents(self, logtype=None, user=None, page=None,
+ start=None, end=None, reverse=False, limit=None):
+ """Iterate all log entries.
+
+ @param logtype: only iterate entries of this type (see wiki
+ documentation for available types, which will include "block",
+ "protect", "rights", "delete", "upload", "move", "import",
+ "patrol", "merge")
+ @param user: only iterate entries that match this user name
+ @param page: only iterate entries affecting this page
+ @param start: only iterate entries from and after this timestamp
+ @param end: only iterate entries up to and through this timestamp
+ @param reverse: if True, iterate oldest entries first (default: newest)
+ @param limit: only iterate up to this many entries
+
+ """
+ if start and end:
+ if reverse:
+ if end < start:
+ raise Error(
+ "logevents: end must be later than start with reverse=True")
+ else:
+ if start < end:
+ raise Error(
+ "logevents: start must be later than end with reverse=False")
+ legen = api.ListGenerator("logevents", site=self)
+ if logtype is not None:
+ legen.request["letype"] = logtype
+ if user is not None:
+ legen.request["leuser"] = user
+ if page is not None:
+ legen.request["letitle"] = page.title(withSection=False)
+ if start is not None:
+ legen.request["lestart"] = start
+ if end is not None:
+ legen.request["leend"] = end
+ if reverse:
+ legen.request["ledir"] = "newer"
+ if isinstance(limit, int):
+ legen.limit = limit
+ return legen
+
+ def recentchanges(self, start=None, end=None, reverse=False, limit=None,
+ namespaces=None, pagelist=None, changetype=None,
+ showMinor=None, showBot=None, showAnon=None,
+ showRedirects=None, showPatrolled=None):
+ """Iterate recent changes.
+
+ @param start: timestamp to start listing from
+ @param end: timestamp to end listing at
+ @param reverse: if True, start with oldest changes (default: newest)
+ @param limit: iterate no more than this number of entries
+ @param namespaces: iterate changes to pages in these namespaces only
+ @type namespaces: list of ints
+ @param pagelist: iterate changes to pages in this list only
+ @param pagelist: list of Pages
+ @param changetype: only iterate changes of this type ("edit" for
+ edits to existing pages, "new" for new pages, "log" for log
+ entries)
+ @param showMinor: if True, only list minor edits; if False (and not
+ None), only list non-minor edits
+ @param showBot: if True, only list bot edits; if False (and not
+ None), only list non-bot edits
+ @param showAnon: if True, only list anon edits; if False (and not
+ None), only list non-anon edits
+ @param showRedirects: if True, only list edits to redirect pages; if
+ False (and not None), only list edits to non-redirect pages
+ @param showPatrolled: if True, only list patrolled edits; if False
+ (and not None), only list non-patrolled edits
+
+ """
+ if start and end:
+ if reverse:
+ if end < start:
+ raise Error(
+ "recentchanges: end must be later than start with reverse=True")
+ else:
+ if start < end:
+ raise Error(
+ "recentchanges: start must be later than end with reverse=False")
+ rcgen = api.ListGenerator("recentchanges", site=self,
+ rcprop="user|comment|timestamp|title|ids"
+ "|redirect|patrolled|loginfo|flags")
+ if start is not None:
+ rcgen.request["rcstart"] = start
+ if end is not None:
+ rcgen.request["rcend"] = end
+ if reverse:
+ rcgen.request["rcdir"] = "newer"
+ if isinstance(limit, int):
+ rcgen.limit = limit
+ if isinstance(namespaces, list):
+ rcgen.request["rcnamespace"] = u"|".join(unicode(ns)
+ for ns in namespaces)
+ elif namespaces is not None:
+ rcgen.request["rcnamespace"] = str(namespaces)
+ if pagelist:
+ rcgen.request["rctitles"] = u"|".join(p.title(withSection=False)
+ for p in pagelist)
+ if changetype:
+ rcgen.request["rctype"] = changetype
+ filters = {'minor': showMinor,
+ 'bot': showBot,
+ 'anon': showAnon,
+ 'redirect': showRedirects,
+ 'patrolled': showPatrolled}
+ rcshow = []
+ for item in filters:
+ if filters[item] is not None:
+ rcshow.append(filters[item] and item or ("!"+item))
+ if rcshow:
+ rcgen.request["rcshow"] = "|".join(rcshow)
+ return rcgen
+
+ @deprecate_arg("number", "limit")
+ def search(self, searchstring, namespaces=None, where="text",
+ getredirects=False, limit=None):
+ """Iterate Pages that contain the searchstring.
+
+ Note that this may include non-existing Pages if the wiki's database
+ table contains outdated entries.
+
+ @param searchstring: the text to search for
+ @type searchstring: unicode
+ @param where: Where to search; value must be "text" or "titles" (many
+ wikis do not support title search)
+ @param namespaces: search only in these namespaces (defaults to 0)
+ @type namespaces: list of ints
+ @param getredirects: if True, include redirects in results
+ @param limit: maximum number of results to iterate
+
+ """
+ if not searchstring:
+ raise Error("search: searchstring cannot be empty")
+ if where not in ("text", "titles"):
+ raise Error("search: unrecognized 'where' value: %s" % where)
+ srgen = api.PageGenerator("search", gsrsearch=searchstring,
+ gsrwhat=where, site=self)
+ if not namespaces:
+ logger.warning("search: namespaces cannot be empty; using [0].")
+ namespaces = [0]
+ if isinstance(namespaces, list):
+ srgen.request["gsrnamespace"] = u"|".join(unicode(ns)
+ for ns in namespaces)
+ else:
+ srgen.request["gsrnamespace"] = str(namespaces)
+ if getredirects:
+ srgen.request["gsrredirects"] = ""
+ if isinstance(limit, int):
+ srgen.limit = limit
+ return srgen
+
+ def usercontribs(self, user=None, userprefix=None, start=None, end=None,
+ reverse=False, limit=None, namespaces=None,
+ showMinor=None):
+ """Iterate contributions by a particular user.
+
+ Iterated values are in the same format as recentchanges.
+
+ @param user: Iterate contributions by this user (name or IP)
+ @param userprefix: Iterate contributions by all users whose names
+ or IPs start with this substring
+ @param start: Iterate contributions starting at this timestamp
+ @param end: Iterate contributions ending at this timestamp
+ @param reverse: Iterate oldest contributions first (default: newest)
+ @param limit: Maximum number of contributions to iterate
+ @param namespaces: Only iterate contributions in these namespaces
+ @type namespaces: list of ints
+ @param showMinor: if True, iterate only minor edits; if False and
+ not None, iterate only non-minor edits (default: iterate both)
+
+ """
+ if not (user or userprefix):
+ raise Error(
+ "usercontribs: either user or userprefix must be non-empty")
+ if start and end:
+ if reverse:
+ if end < start:
+ raise Error(
+ "usercontribs: end must be later than start with reverse=True")
+ else:
+ if start < end:
+ raise Error(
+ "usercontribs: start must be later than end with reverse=False")
+ ucgen = api.ListGenerator("usercontribs", site=self,
+ ucprop="ids|title|timestamp|comment|flags")
+ if user:
+ ucgen.request["ucuser"] = user
+ if userprefix:
+ ucgen.request["ucuserprefix"] = userprefix
+ if start is not None:
+ ucgen.request["ucstart"] = start
+ if end is not None:
+ ucgen.request["ucend"] = end
+ if reverse:
+ ucgen.request["ucdir"] = "newer"
+ if isinstance(limit, int):
+ ucgen.limit = limit
+ if isinstance(namespaces, list):
+ ucgen.request["ucnamespace"] = u"|".join(unicode(ns)
+ for ns in namespaces)
+ elif namespaces is not None:
+ ucgen.request["ucnamespace"] = str(namespaces)
+ if showMinor is not None:
+ ucgen.request["ucshow"] = showMinor and "minor" or "!minor"
+ return ucgen
+
+ def watchlist_revs(self, start=None, end=None, reverse=False,
+ namespaces=None, showMinor=None, showBot=None,
+ showAnon=None, limit=None):
+ """Iterate revisions to pages on the bot user's watchlist.
+
+ Iterated values will be in same format as recentchanges.
+
+ @param start: Iterate revisions starting at this timestamp
+ @param end: Iterate revisions ending at this timestamp
+ @param reverse: Iterate oldest revisions first (default: newest)
+ @param namespaces: only iterate revisions to pages in these
+ namespaces (default: all)
+ @type namespaces: list of ints
+ @param showMinor: if True, only list minor edits; if False (and not
+ None), only list non-minor edits
+ @param showBot: if True, only list bot edits; if False (and not
+ None), only list non-bot edits
+ @param showAnon: if True, only list anon edits; if False (and not
+ None), only list non-anon edits
+ @param limit: Maximum number of revisions to iterate
+
+ """
+ if start and end:
+ if reverse:
+ if end < start:
+ raise Error(
+ "watchlist_revs: end must be later than start with reverse=True")
+ else:
+ if start < end:
+ raise Error(
+ "watchlist_revs: start must be later than end with reverse=False")
+ wlgen = api.ListGenerator("watchlist", wlallrev="", site=self,
+ wlprop="user|comment|timestamp|title|ids|flags")
+ #TODO: allow users to ask for "patrol" as well?
+ if start is not None:
+ wlgen.request["wlstart"] = start
+ if end is not None:
+ wlgen.request["wlend"] = end
+ if reverse:
+ wlgen.request["wldir"] = "newer"
+ if isinstance(limit, int):
+ wlgen.limit = limit
+ if isinstance(namespaces, list):
+ wlgen.request["wlnamespace"] = u"|".join(unicode(ns)
+ for ns in namespaces)
+ elif namespaces is not None:
+ wlgen.request["wlnamespace"] = str(namespaces)
+ filters = {'minor': showMinor,
+ 'bot': showBot,
+ 'anon': showAnon}
+ wlshow = []
+ for item in filters:
+ if filters[item] is not None:
+ wlshow.append(filters[item] and item or ("!"+item))
+ if wlshow:
+ wlgen.request["wlshow"] = "|".join(wlshow)
+ return wlgen
+
+ def deletedrevs(self, page, start=None, end=None, reverse=None, limit=None,
+ get_text=False):
+ """Iterate deleted revisions.
+
+ Each value returned by the iterator will be a dict containing the
+ 'title' and 'ns' keys for a particular Page and a 'revisions' key
+ whose value is a list of revisions in the same format as
+ recentchanges (plus a 'content' element if requested). If get_text
+ is true, the toplevel dict will contain a 'token' key as well.
+
+ @param page: The page to check for deleted revisions
+ @param start: Iterate revisions starting at this timestamp
+ @param end: Iterate revisions ending at this timestamp
+ @param reverse: Iterate oldest revisions first (default: newest)
+ @param limit: Iterate no more than this number of revisions.
+ @param get_text: If True, retrieve the content of each revision and
+ an undelete token
+
+ """
+ if start and end:
+ if reverse:
+ if end < start:
+ raise Error(
+"deletedrevs: end must be later than start with reverse=True")
+ else:
+ if start < end:
+ raise Error(
+"deletedrevs: start must be later than end with reverse=False")
+ if not self.logged_in():
+ self.login()
+ if "deletedhistory" not in self.userinfo['rights']:
+ try:
+ self.login(True)
+ except NoUsername:
+ pass
+ if "deletedhistory" not in self.userinfo['rights']:
+ raise Error(
+"deletedrevs: User:%s not authorized to access deleted revisions."
+ % self.user())
+ if get_text:
+ if "undelete" not in self.userinfo['rights']:
+ try:
+ self.login(True)
+ except NoUsername:
+ pass
+ if "undelete" not in self.userinfo['rights']:
+ raise Error(
+"deletedrevs: User:%s not authorized to view deleted content."
+ % self.user())
+
+ drgen = api.ListGenerator("deletedrevs", site=self,
+ titles=page.title(withSection=False),
+ drprop="revid|user|comment|minor")
+ if get_text:
+ drgen.request['drprop'] = drgen.request['drprop'] + "|content|token"
+ if start is not None:
+ drgen.request["drstart"] = start
+ if end is not None:
+ drgen.request["drend"] = end
+ if reverse:
+ drgen.request["drdir"] = "newer"
+ if isinstance(limit, int):
+ drgen.limit = limit
+ return drgen
+
+ def users(self, usernames):
+ """Iterate info about a list of users by name or IP.
+
+ @param usernames: a list of user names
+ @type usernames: list, or other iterable, of unicodes
+
+ """
+ if not isinstance(usernames, basestring):
+ usernames = u"|".join(usernames)
+ usgen = api.ListGenerator("users", ususers=usernames, site=self,
+ usprop="blockinfo|groups|editcount|registration")
+ return usgen
+
+ def randompages(self, limit=1, namespaces=None, redirects=False):
+ """Iterate a number of random pages.
+
+ Pages are listed in a fixed sequence, only the starting point is
+ random.
+
+ @param limit: the maximum number of pages to iterate (default: 1)
+ @param namespaces: only iterate pages in these namespaces.
+ @param redirects: if True, include only redirect pages in results
+ (default: include only non-redirects)
+
+ """
+ rngen = api.PageGenerator("random", site=self)
+ rngen.limit = limit
+ if isinstance(namespaces, list):
+ rngen.request["grnnamespace"] = u"|".join(unicode(ns)
+ for ns in namespaces)
+ elif namespaces is not None:
+ rngen.request["grnnamespace"] = str(namespaces)
+ if redirects:
+ rngen.request["grnredirect"] = ""
+ return rngen
+
+ # catalog of editpage error codes, for use in generating messages
+ _ep_errors = {
+ "noapiwrite": "API editing not enabled on %(site)s wiki",
+ "writeapidenied":
+"User %(user)s is not authorized to edit on %(site)s wiki",
+ "protectedtitle":
+"Title %(title)s is protected against creation on %(site)s",
+ "cantcreate":
+"User %(user)s not authorized to create new pages on %(site)s wiki",
+ "cantcreate-anon":
+"""Bot is not logged in, and anon users are not authorized to create new pages
+on %(site)s wiki""",
+ "articleexists": "Page %(title)s already exists on %(site)s wiki",
+ "noimageredirect-anon":
+"""Bot is not logged in, and anon users are not authorized to create image
+redirects on %(site)s wiki""",
+ "noimageredirect":
+"User %(user)s not authorized to create image redirects on %(site)s wiki",
+ "spamdetected":
+"Edit to page %(title)s rejected by spam filter due to content:\n",
+ "filtered": "%(info)s",
+ "contenttoobig": "%(info)s",
+ "noedit-anon":
+"""Bot is not logged in, and anon users are not authorized to edit on
+%(site)s wiki""",
+ "noedit": "User %(user)s not authorized to edit pages on %(site)s wiki",
+ "pagedeleted":
+"Page %(title)s has been deleted since last retrieved from %(site)s wiki",
+ "editconflict": "Page %(title)s not saved due to edit conflict.",
+ }
+
+ def editpage(self, page, summary, minor=True, notminor=False,
+ recreate=True, createonly=False, watch=False, unwatch=False):
+ """Submit an edited Page object to be saved to the wiki.
+
+ @param page: The Page to be saved; its .text property will be used
+ as the new text to be saved to the wiki
+ @param token: the edit token retrieved using Site.token()
+ @param summary: the edit summary (required!)
+ @param minor: if True (default), mark edit as minor
+ @param notminor: if True, override account preferences to mark edit
+ as non-minor
+ @param recreate: if True (default), create new page even if this
+ title has previously been deleted
+ @param createonly: if True, raise an error if this title already
+ exists on the wiki
+ @param watch: if True, add this Page to bot's watchlist
+ @param unwatch: if True, remove this Page from bot's watchlist if
+ possible
+ @return: True if edit succeeded, False if it failed
+
+ """
+ text = page.text
+ if not text:
+ raise Error("editpage: no text to be saved")
+ try:
+ lastrev = page.latestRevision()
+ except NoPage:
+ lastrev = None
+ if not recreate:
+ raise Error("Page %s does not exist on %s wiki."
+ % (page.title(withSection=False), self))
+ token = self.token(page, "edit")
+ self.lock_page(page)
+ if lastrev is not None and page.latestRevision() != lastrev:
+ raise Error("editpage: Edit conflict detected; saving aborted.")
+ req = api.Request(site=self, action="edit",
+ title=page.title(withSection=False),
+ text=text, token=token, summary=summary)
+## if lastrev is not None:
+## req["basetimestamp"] = page._revisions[lastrev].timestamp
+ if minor:
+ req['minor'] = ""
+ elif notminor:
+ req['notminor'] = ""
+ if 'bot' in self.userinfo['groups']:
+ req['bot'] = ""
+ if recreate:
+ req['recreate'] = ""
+ if createonly:
+ req['createonly'] = ""
+ if watch:
+ req['watch'] = ""
+ elif unwatch:
+ req['unwatch'] = ""
+## FIXME: API gives 'badmd5' error
+## md5hash = md5()
+## md5hash.update(urllib.quote_plus(text.encode(self.encoding())))
+## req['md5'] = md5hash.digest()
+ while True:
+ try:
+ result = req.submit()
+ logger.debug("editpage response: %s" % result)
+ except api.APIError, err:
+ self.unlock_page(page)
+ if err.code.endswith("anon") and self.logged_in():
+ logger.debug(
+"editpage: received '%s' even though bot is logged in" % err.code)
+ errdata = {
+ 'site': self,
+ 'title': page.title(withSection=False),
+ 'user': self.user(),
+ 'info': err.info
+ }
+ if err.code == "spamdetected":
+ raise SpamfilterError(self._ep_errors[err.code] % errdata
+ + err.info[ err.info.index("fragment: ") + 9: ])
+
+ if err.code == "editconflict":
+ raise EditConflict(self._ep_errors[err.code] % errdata)
+ if err.code in self._ep_errors:
+ raise Error(self._ep_errors[err.code] % errdata)
+ logger.debug("editpage: Unexpected error code '%s' received."
+ % err.code)
+ raise
+ assert ("edit" in result and "result" in result["edit"]), result
+ if result["edit"]["result"] == "Success":
+ self.unlock_page(page)
+ if "nochange" in result["edit"]:
+ # null edit, page not changed
+ # TODO: do we want to notify the user of this?
+ return True
+ page._revid = result["edit"]["newrevid"]
+ # see http://www.mediawiki.org/wiki/API:Wikimania_2006_API_discussion#Notes
+ # not safe to assume that saved text is the same as sent
+ self.loadrevisions(page, getText=True)
+ return True
+ elif result["edit"]["result"] == "Failure":
+ if "captcha" in result["edit"]:
+ captcha = result["edit"]["captcha"]
+ req['captchaid'] = captcha['id']
+ if captcha["type"] == "math":
+ req['captchaword'] = input(captcha["question"])
+ continue
+ elif "url" in captcha:
+ webbrowser.open(url)
+ req['captchaword'] = cap_answerwikipedia.input(
+"Please view CAPTCHA in your browser, then type answer here:")
+ continue
+ else:
+ self.unlock_page(page)
+ logger.error(
+"editpage: unknown CAPTCHA response %s, page not saved"
+ % captcha)
+ return False
+ else:
+ self.unlock_page(page)
+ logger.error("editpage: unknown failure reason %s"
+ % str(result))
+ return False
+ else:
+ self.unlock_page(page)
+ logger.error(
+"editpage: Unknown result code '%s' received; page not saved"
+ % result["edit"]["result"])
+ logger.error(str(result))
+ return False
+
+ # catalog of move errors for use in error messages
+ _mv_errors = {
+ "noapiwrite": "API editing not enabled on %(site)s wiki",
+ "writeapidenied":
+"User %(user)s is not authorized to edit on %(site)s wiki",
+ "nosuppress":
+"User %(user)s is not authorized to move pages without creating redirects",
+ "cantmove-anon":
+"""Bot is not logged in, and anon users are not authorized to move pages on
+%(site)s wiki""",
+ "cantmove":
+"User %(user)s is not authorized to move pages on %(site)s wiki",
+ "immobilenamespace":
+"Pages in %(oldnamespace)s namespace cannot be moved on %(site)s wiki",
+ "articleexists":
+"Cannot move because page [[%(newtitle)s]] already exists on %(site)s wiki",
+ "protectedpage":
+"Page [[%(oldtitle)s]] is protected against moving on %(site)s wiki",
+ "protectedtitle":
+"Page [[%(newtitle)s]] is protected against creation on %(site)s wiki",
+ "nonfilenamespace":
+"Cannot move a file to %(newnamespace)s namespace on %(site)s wiki",
+ "filetypemismatch":
+"[[%(newtitle)s]] file extension does not match content of [[%(oldtitle)s]]"
+ }
+
+ def movepage(self, page, newtitle, summary, movetalk=True,
+ noredirect=False):
+ """Move a Page to a new title.
+
+ @param page: the Page to be moved (must exist)
+ @param newtitle: the new title for the Page
+ @type newtitle: unicode
+ @param summary: edit summary (required!)
+ @param movetalk: if True (default), also move the talk page if possible
+ @param noredirect: if True, suppress creation of a redirect from the
+ old title to the new one
+ @return: Page object with the new title
+
+ """
+ oldtitle = page.title(withSection=False)
+ newlink = pywikibot.Link(newtitle, self)
+ if newlink.namespace:
+ newtitle = self.namespace(newlink.namespace) + ":" + newlink.title
+ else:
+ newtitle = newlink.title
+ if oldtitle == newtitle:
+ raise Error("Cannot move page %s to its own title."
+ % oldtitle)
+ if not page.exists():
+ raise Error("Cannot move page %s because it does not exist on %s."
+ % (oldtitle, self))
+ token = self.token(page, "move")
+ self.lock_page(page)
+ req = api.Request(site=self, action="move", to=newtitle,
+ token=token, reason=summary)
+ req['from'] = oldtitle # "from" is a python keyword
+ if movetalk:
+ req['movetalk'] = ""
+ if noredirect:
+ req['noredirect'] = ""
+ try:
+ result = req.submit()
+ logger.debug("movepage response: %s" % result)
+ except api.APIError, err:
+ if err.code.endswith("anon") and self.logged_in():
+ logger.debug(
+"movepage: received '%s' even though bot is logged in" % err.code)
+ errdata = {
+ 'site': self,
+ 'oldtitle': oldtitle,
+ 'oldnamespace': self.namespace(page.namespace()),
+ 'newtitle': newtitle,
+ 'newnamespace': self.namespace(newlink.namespace),
+ 'user': self.user(),
+ }
+ if err.code in self._mv_errors:
+ raise Error(self._mv_errors[err.code] % errdata)
+ logger.debug("movepage: Unexpected error code '%s' received."
+ % err.code)
+ raise
+ finally:
+ self.unlock_page(page)
+ if "move" not in result:
+ logger.error("movepage: %s" % result)
+ raise Error("movepage: unexpected response")
+ # TODO: Check for talkmove-error messages
+ if "talkmove-error-code" in result["move"]:
+ logger.warning(u"movepage: Talk page %s not moved"
+ % (page.toggleTalkPage().title(asLink=True)))
+ return pywikibot.Page(page, newtitle)
+
+ # catalog of rollback errors for use in error messages
+ _rb_errors = {
+ "noapiwrite":
+ "API editing not enabled on %(site)s wiki",
+ "writeapidenied":
+ "User %(user)s not allowed to edit through the API",
+ "alreadyrolled":
+ "Page [[%(title)s]] already rolled back; action aborted.",
+ } # other errors shouldn't arise because we check for those errors
+
+ def rollbackpage(self, page, summary=u''):
+ """Roll back page to version before last user's edits.
+
+ As a precaution against errors, this method will fail unless
+ the page history contains at least two revisions, and at least
+ one that is not by the same user who made the last edit.
+
+ @param page: the Page to be rolled back (must exist)
+ @param summary: edit summary (defaults to a standardized message)
+
+ """
+ if len(page._revisions) < 2:
+ raise pywikibot.Error(
+ u"Rollback of %s aborted; load revision history first."
+ % page.title(asLink=True))
+ last_rev = page._revisions[page.latestRevision()]
+ last_user = last_rev.user
+ for rev in sorted(page._revisions.keys(), reverse=True):
+ # start with most recent revision first
+ if rev.user != last_user:
+ prev_user = rev.user
+ break
+ else:
+ raise pywikibot.Error(
+ u"Rollback of %s aborted; only one user in revision history."
+ % page.title(asLink=True))
+ summary = summary or (
+u"Reverted edits by [[Special:Contributions/%(last_user)s|%(last_user)s]] "
+u"([[User talk:%(last_user)s|Talk]]) to last version by %(prev_user)s"
+ % locals())
+ token = self.token(page, "rollback")
+ self.lock_page(page)
+ req = api.Request(site=self, action="rollback",
+ title=page.title(withSection=False),
+ user=last_user,
+ token=token)
+ try:
+ result = req.submit()
+ except api.APIError, err:
+ errdata = {
+ 'site': self,
+ 'title': page.title(withSection=False),
+ 'user': self.user(),
+ }
+ if err.code in self._rb_errors:
+ raise Error(self._rb_errors[err.code] % errdata)
+ logger.debug("rollback: Unexpected error code '%s' received."
+ % err.code)
+ raise
+ finally:
+ self.unlock_page(page)
+
+ # catalog of delete errors for use in error messages
+ _dl_errors = {
+ "noapiwrite":
+ "API editing not enabled on %(site)s wiki",
+ "writeapidenied":
+ "User %(user)s not allowed to edit through the API",
+ "permissiondenied":
+ "User %(user)s not authorized to delete pages on %(site)s wiki.",
+ "cantdelete":
+ "Could not delete [[%(title)s]]. Maybe it was deleted already.",
+ } # other errors shouldn't occur because of pre-submission checks
+
+ def deletepage(self, page, summary):
+ """Delete page from the wiki. Requires appropriate privilege level.
+
+ @param page: Page to be deleted.
+ @param summary: Edit summary (required!).
+
+ """
+ try:
+ self.login(sysop=True)
+ except pywikibot.Error, e:
+ raise Error("delete: Unable to login as sysop (%s)"
+ % e.__class__.__name__)
+ if not self.logged_in(sysop=True):
+ raise Error("delete: Unable to login as sysop")
+ token = self.token("delete")
+ req = api.Request(site=self, action="delete", token=token,
+ title=page.title(withSection=False),
+ reason=summary)
+ try:
+ result = req.submit()
+ except api.APIError, err:
+ errdata = {
+ 'site': self,
+ 'title': page.title(withSection=False),
+ 'user': self.user(),
+ }
+ if err.code in self._dl_errors:
+ raise Error(self._dl_errors[err.code] % errdata)
+ logger.debug("delete: Unexpected error code '%s' received."
+ % err.code)
+ raise
+ finally:
+ self.unlock_page(page)
+
+ # TODO: implement undelete
+
+ # TODO: implement patrol
+
+ def linksearch(self, siteurl, limit=500):
+ """Backwards-compatible interface to exturlusage()"""
+ return self.exturlusage(siteurl, limit=limit)
+
+ @deprecate_arg("repeat", None)
+ def newimages(self, number=100, lestart=None, leend=None, leuser=None,
+ letitle=None):
+ """Yield ImagePages from most recent uploads"""
+ return self.logevents(logtype="upload", limit=number, start=lestart,
+ end=leend, user=leuser, title=letitle)
+
+ def getImagesFromAnHash(self, hash_found=None):
+ """Return all images that have the same hash.
+
+ Useful to find duplicates or nowcommons.
+
+ NOTE: it returns also the image itself, if you don't want it, just
+ filter the list returned.
+
+ NOTE 2: it returns the image title WITHOUT the image namespace.
+
+ """
+ if hash_found == None: # If the hash is none return None and not continue
+ return None
+ return [image.title(withNamespace=False)
+ for image in self.allimages(sha1=hash_found)]
+
+
+#### METHODS NOT IMPLEMENTED YET ####
+class NotImplementedYet:
+
+ # TODO: is this needed any more? can it be obtained from the http module?
+ def cookies(self, sysop = False):
+ """Return a string containing the user's current cookies."""
+ self._loadCookies(sysop = sysop)
+ index = self._userIndex(sysop)
+ return self._cookies[index]
+
+ def _loadCookies(self, sysop = False):
+ """Retrieve session cookies for login"""
+ index = self._userIndex(sysop)
+ if self._cookies[index] is not None:
+ return
+ try:
+ if sysop:
+ try:
+ username = config.sysopnames[self.family.name
+ ][self.code]
+ except KeyError:
+ raise NoUsername("""\
+You tried to perform an action that requires admin privileges, but you haven't
+entered your sysop name in your user-config.py. Please add
+sysopnames['%s']['%s']='name' to your user-config.py"""
+ % (self.family.name, self.code))
+ else:
+ username = pywikiobt.config2.usernames[self.family.name
+ ][self.code]
+ except KeyError:
+ self._cookies[index] = None
+ self._isLoggedIn[index] = False
+ else:
+ tmp = '%s-%s-%s-login.data' % (
+ self.family.name, self.code, username)
+ fn = config.datafilepath('login-data', tmp)
+ if not os.path.exists(fn):
+ self._cookies[index] = None
+ self._isLoggedIn[index] = False
+ else:
+ f = open(fn)
+ self._cookies[index] = '; '.join([x.strip() for x in f.readlines()])
+ f.close()
+
+ # THESE ARE FUNCTIONS NOT YET IMPLEMENTED IN THE API
+ # TODO: avoid code duplication for the following methods
+ def newpages(self, number = 10, get_redirect = False, repeat = False):
+ """Yield new articles (as Page objects) from Special:Newpages.
+
+ Starts with the newest article and fetches the number of articles
+ specified in the first argument. If repeat is True, it fetches
+ Newpages again. If there is no new page, it blocks until there is
+ one, sleeping between subsequent fetches of Newpages.
+
+ The objects yielded are tuples composed of the Page object,
+ timestamp (unicode), length (int), an empty unicode string, username
+ or IP address (str), comment (unicode).
+
+ """
+ # TODO: in recent MW versions Special:Newpages takes a namespace parameter,
+ # and defaults to 0 if not specified.
+ # TODO: Detection of unregistered users is broken
+ # TODO: Repeat mechanism doesn't make much sense as implemented;
+ # should use both offset and limit parameters, and have an
+ # option to fetch older rather than newer pages
+ seen = set()
+ while True:
+ path = self.newpages_address(n=number)
+ # The throttling is important here, so always enabled.
+ get_throttle()
+ html = self.getUrl(path)
+
+ entryR = re.compile(
+'<li[^>]*>(?P<date>.+?) \S*?<a href=".+?"'
+' title="(?P<title>.+?)">.+?</a>.+?[\(\[](?P<length>[\d,.]+)[^\)\]]*[\)\]]'
+' .?<a href=".+?" title=".+?:(?P<username>.+?)">'
+ )
+ for m in entryR.finditer(html):
+ date = m.group('date')
+ title = m.group('title')
+ title = title.replace('"', '"')
+ length = int(re.sub("[,.]", "", m.group('length')))
+ loggedIn = u''
+ username = m.group('username')
+ comment = u''
+
+ if title not in seen:
+ seen.add(title)
+ page = Page(self, title)
+ yield page, date, length, loggedIn, username, comment
+ if not repeat:
+ break
+
+ def longpages(self, number = 10, repeat = False):
+ """Yield Pages from Special:Longpages.
+
+ Return values are a tuple of Page object, length(int).
+
+ """
+ #TODO: should use offset and limit parameters; 'repeat' as now
+ # implemented is fairly useless
+ # this comment applies to all the XXXXpages methods following, as well
+ seen = set()
+ while True:
+ path = self.longpages_address(n=number)
+ get_throttle()
+ html = self.getUrl(path)
+ entryR = re.compile(ur'<li>\(<a href=".+?" title=".+?">hist</a>\) <a href=".+?" title="(?P<title>.+?)">.+?</a> \[(?P<length>\d+)(.+?)\]</li>')
+ for m in entryR.finditer(html):
+ title = m.group('title')
+ length = int(m.group('length'))
+ if title not in seen:
+ seen.add(title)
+ page = Page(self, title)
+ yield page, length
+ if not repeat:
+ break
+
+ def shortpages(self, number = 10, repeat = False):
+ """Yield Pages and lengths from Special:Shortpages."""
+ throttle = True
+ seen = set()
+ while True:
+ path = self.shortpages_address(n = number)
+ get_throttle()
+ html = self.getUrl(path)
+ entryR = re.compile(ur'<li>\(<a href=".+?" title=".+?">hist</a>\) <a href=".+?" title="(?P<title>.+?)">.+?</a> \[(?P<length>\d+)(.+?)\]</li>')
+ for m in entryR.finditer(html):
+ title = m.group('title')
+ length = int(m.group('length'))
+
+ if title not in seen:
+ seen.add(title)
+ page = Page(self, title)
+ yield page, length
+ if not repeat:
+ break
+
+ def deadendpages(self, number = 10, repeat = False):
+ """Yield Page objects retrieved from Special:Deadendpages."""
+ seen = set()
+ while True:
+ path = self.deadendpages_address(n=number)
+ get_throttle()
+ html = self.getUrl(path)
+ entryR = re.compile(
+ '<li><a href=".+?" title="(?P<title>.+?)">.+?</a></li>')
+ for m in entryR.finditer(html):
+ title = m.group('title')
+
+ if title not in seen:
+ seen.add(title)
+ page = Page(self, title)
+ yield page
+ if not repeat:
+ break
+
+ def ancientpages(self, number = 10, repeat = False):
+ """Yield Pages, datestamps from Special:Ancientpages."""
+ seen = set()
+ while True:
+ path = self.ancientpages_address(n=number)
+ get_throttle()
+ html = self.getUrl(path)
+ entryR = re.compile(
+'<li><a href=".+?" title="(?P<title>.+?)">.+?</a> (?P<date>.+?)</li>')
+ for m in entryR.finditer(html):
+ title = m.group('title')
+ date = m.group('date')
+ if title not in seen:
+ seen.add(title)
+ page = Page(self, title)
+ yield page, date
+ if not repeat:
+ break
+
+ def lonelypages(self, number = 10, repeat = False):
+ """Yield Pages retrieved from Special:Lonelypages."""
+ throttle = True
+ seen = set()
+ while True:
+ path = self.lonelypages_address(n=number)
+ get_throttle()
+ html = self.getUrl(path)
+ entryR = re.compile(
+ '<li><a href=".+?" title="(?P<title>.+?)">.+?</a></li>')
+ for m in entryR.finditer(html):
+ title = m.group('title')
+
+ if title not in seen:
+ seen.add(title)
+ page = Page(self, title)
+ yield page
+ if not repeat:
+ break
+
+ def unwatchedpages(self, number = 10, repeat = False):
+ """Yield Pages from Special:Unwatchedpages (requires Admin privileges)."""
+ seen = set()
+ while True:
+ path = self.unwatchedpages_address(n=number)
+ get_throttle()
+ html = self.getUrl(path, sysop = True)
+ entryR = re.compile(
+ '<li><a href=".+?" title="(?P<title>.+?)">.+?</a>.+?</li>')
+ for m in entryR.finditer(html):
+ title = m.group('title')
+ if title not in seen:
+ seen.add(title)
+ page = Page(self, title)
+ yield page
+ if not repeat:
+ break
+
+ def uncategorizedcategories(self, number = 10, repeat = False):
+ """Yield Categories from Special:Uncategorizedcategories."""
+ import catlib
+ seen = set()
+ while True:
+ path = self.uncategorizedcategories_address(n=number)
+ get_throttle()
+ html = self.getUrl(path)
+ entryR = re.compile(
+ '<li><a href=".+?" title="(?P<title>.+?)">.+?</a></li>')
+ for m in entryR.finditer(html):
+ title = m.group('title')
+ if title not in seen:
+ seen.add(title)
+ page = catlib.Category(self, title)
+ yield page
+ if not repeat:
+ break
+
+ def newimages(self, number = 10, repeat = False):
+ """Yield ImagePages from Special:Log&type=upload"""
+
+ seen = set()
+ regexp = re.compile('<li[^>]*>(?P<date>.+?)\s+<a href=.*?>(?P<user>.+?)</a>\s+\(.+?</a>\).*?<a href=".*?"(?P<new> class="new")? title="(?P<image>.+?)"\s*>(?:.*?<span class="comment">(?P<comment>.*?)</span>)?', re.UNICODE)
+
+ while True:
+ path = self.log_address(number, mode = 'upload')
+ get_throttle()
+ html = self.getUrl(path)
+
+ for m in regexp.finditer(html):
+ image = m.group('image')
+
+ if image not in seen:
+ seen.add(image)
+
+ if m.group('new'):
+ output(u"Image \'%s\' has been deleted." % image)
+ continue
+
+ date = m.group('date')
+ user = m.group('user')
+ comment = m.group('comment') or ''
+
+ yield ImagePage(self, image), date, user, comment
+ if not repeat:
+ break
+
+ def uncategorizedimages(self, number = 10, repeat = False):
+ """Yield ImagePages from Special:Uncategorizedimages."""
+ seen = set()
+ ns = self.image_namespace()
+ entryR = re.compile(
+ '<a href=".+?" title="(?P<title>%s:.+?)">.+?</a>' % ns)
+ while True:
+ path = self.uncategorizedimages_address(n=number)
+ get_throttle()
+ html = self.getUrl(path)
+ for m in entryR.finditer(html):
+ title = m.group('title')
+ if title not in seen:
+ seen.add(title)
+ page = ImagePage(self, title)
+ yield page
+ if not repeat:
+ break
+
+ def uncategorizedpages(self, number = 10, repeat = False):
+ """Yield Pages from Special:Uncategorizedpages."""
+ seen = set()
+ while True:
+ path = self.uncategorizedpages_address(n=number)
+ get_throttle()
+ html = self.getUrl(path)
+ entryR = re.compile(
+ '<li><a href=".+?" title="(?P<title>.+?)">.+?</a></li>')
+ for m in entryR.finditer(html):
+ title = m.group('title')
+
+ if title not in seen:
+ seen.add(title)
+ page = Page(self, title)
+ yield page
+ if not repeat:
+ break
+
+ def unusedcategories(self, number = 10, repeat = False):
+ """Yield Category objects from Special:Unusedcategories."""
+ import catlib
+ seen = set()
+ while True:
+ path = self.unusedcategories_address(n=number)
+ get_throttle()
+ html = self.getUrl(path)
+ entryR = re.compile('<li><a href=".+?" title="(?P<title>.+?)">.+?</a></li>')
+ for m in entryR.finditer(html):
+ title = m.group('title')
+
+ if title not in seen:
+ seen.add(title)
+ page = catlib.Category(self, title)
+ yield page
+ if not repeat:
+ break
+
+ def unusedfiles(self, number = 10, repeat = False, extension = None):
+ """Yield ImagePage objects from Special:Unusedimages."""
+ seen = set()
+ ns = self.image_namespace()
+ entryR = re.compile(
+ '<a href=".+?" title="(?P<title>%s:.+?)">.+?</a>' % ns)
+ while True:
+ path = self.unusedfiles_address(n=number)
+ get_throttle()
+ html = self.getUrl(path)
+ for m in entryR.finditer(html):
+ fileext = None
+ title = m.group('title')
+ if extension:
+ fileext = title[len(title)-3:]
+ if title not in seen and fileext == extension:
+ ## Check whether the media is used in a Proofread page
+ # code disabled because it slows this method down, and
+ # because it is unclear what it's supposed to do.
+ #basename = title[6:]
+ #page = Page(self, 'Page:' + basename)
+
+ #if not page.exists():
+ seen.add(title)
+ image = ImagePage(self, title)
+ yield image
+ if not repeat:
+ break
+
+ def withoutinterwiki(self, number=10, repeat=False):
+ """Yield Pages without language links from Special:Withoutinterwiki."""
+ seen = set()
+ while True:
+ path = self.withoutinterwiki_address(n=number)
+ get_throttle()
+ html = self.getUrl(path)
+ entryR = re.compile('<li><a href=".+?" title="(?P<title>.+?)">.+?</a></li>')
+ for m in entryR.finditer(html):
+ title = m.group('title')
+ if title not in seen:
+ seen.add(title)
+ page = Page(self, title)
+ yield page
+ if not repeat:
+ break
+
+ def linksearch(self, siteurl):
+ """Yield Pages from results of Special:Linksearch for 'siteurl'."""
+ if siteurl.startswith('*.'):
+ siteurl = siteurl[2:]
+ output(u'Querying [[Special:Linksearch]]...')
+ cache = []
+ for url in [siteurl, '*.' + siteurl]:
+ path = self.linksearch_address(url)
+ get_throttle()
+ html = self.getUrl(path)
+ loc = html.find('<div class="mw-spcontent">')
+ if loc > -1:
+ html = html[loc:]
+ loc = html.find('<div class="printfooter">')
+ if loc > -1:
+ html = html[:loc]
+ R = re.compile('title ?=\"(.*?)\"')
+ for title in R.findall(html):
+ if not siteurl in title:
+ # the links themselves have similar form
+ if title in cache:
+ continue
+ else:
+ cache.append(title)
+ yield Page(self, title)
+
Property changes on: branches/rewrite/pywikibot/site.py
___________________________________________________________________
Added: svn:keywords
+ Author Date Id Revision
Added: svn:eol-style
+ native
Modified: branches/rewrite/pywikibot/textlib.py
===================================================================
--- branches/rewrite/pywikibot/textlib.py 2008-12-16 19:34:48 UTC (rev 6155)
+++ branches/rewrite/pywikibot/textlib.py 2008-12-16 19:40:20 UTC (rev 6156)
@@ -1,675 +1,675 @@
-# -*- coding: utf-8 -*-
-"""
-Functions for manipulating wiki-text.
-
-Unless otherwise noted, all functions take a unicode string as the argument
-and return a unicode string.
-
-"""
-#
-# (C) Pywikipedia bot team, 2008
-#
-# Distributed under the terms of the MIT license.
-#
-__version__ = '$Id: $'
-
-
-import pywikibot
-import re
-
-
-def unescape(s):
- """Replace escaped HTML-special characters by their originals"""
- if '&' not in s:
- return s
- s = s.replace("<", "<")
- s = s.replace(">", ">")
- s = s.replace("'", "'")
- s = s.replace(""", '"')
- s = s.replace("&", "&") # Must be last
- return s
-
-
-def replaceExcept(text, old, new, exceptions, caseInsensitive=False,
- allowoverlap=False, marker = '', site = None):
- """
- Return text with 'old' replaced by 'new', ignoring specified types of text.
-
- Skips occurences of 'old' within exceptions; e.g., within nowiki tags or
- HTML comments. If caseInsensitive is true, then use case insensitive
- regex matching. If allowoverlap is true, overlapping occurences are all
- replaced (watch out when using this, it might lead to infinite loops!).
-
- Parameters:
- text - a unicode string
- old - a compiled regular expression
- new - a unicode string (which can contain regular
- expression references), or a function which takes
- a match object as parameter. See parameter repl of
- re.sub().
- exceptions - a list of strings which signal what to leave out,
- e.g. ['math', 'table', 'template']
- caseInsensitive - a boolean
- marker - a string that will be added to the last replacement;
- if nothing is changed, it is added at the end
-
- """
- if site is None:
- site = pywikibot.getSite()
-
- exceptionRegexes = {
- 'comment': re.compile(r'(?s)<!--.*?-->'),
- # section headers
- 'header': re.compile(r'\r\n=+.+=+ *\r\n'),
- 'includeonly': re.compile(r'(?is)<includeonly>.*?</includeonly>'),
- 'math': re.compile(r'(?is)<math>.*?</math>'),
- 'noinclude': re.compile(r'(?is)<noinclude>.*?</noinclude>'),
- # wiki tags are ignored inside nowiki tags.
- 'nowiki': re.compile(r'(?is)<nowiki>.*?</nowiki>'),
- # preformatted text
- 'pre': re.compile(r'(?ism)<pre>.*?</pre>'),
- 'source': re.compile(r'(?is)<source .*?</source>'),
- # inline references
- 'ref': re.compile(r'(?ism)<ref[ >].*?</ref>'),
- 'timeline': re.compile(r'(?is)<timeline>.*?</timeline>'),
- # lines that start with a space are shown in a monospace font and
- # have whitespace preserved.
- 'startspace': re.compile(r'(?m)^ (.*?)$'),
- # tables often have whitespace that is used to improve wiki
- # source code readability.
- # TODO: handle nested tables.
- 'table': re.compile(r'(?ims)^{\|.*?^\|}|<table>.*?</table>'),
- # templates with parameters often have whitespace that is used to
- # improve wiki source code readability.
- # 'template': re.compile(r'(?s){{.*?}}'),
- # The regex above fails on nested templates. This regex can handle
- # templates cascaded up to level 3, but no deeper. For arbitrary
- # depth, we'd need recursion which can't be done in Python's re.
- # After all, the language of correct parenthesis words is not regular.
- 'template': re.compile(r'(?s){{(({{(({{.*?}})|.)*}})|.)*}}'),
- 'hyperlink': compileLinkR(),
- 'gallery': re.compile(r'(?is)<gallery.*?>.*?</gallery>'),
- # this matches internal wikilinks, but also interwiki, categories, and
- # images.
- 'link': re.compile(r'\[\[[^\]\|]*(\|[^\]]*)?\]\]'),
- 'interwiki': re.compile(r'(?i)\[\[(%s)\s?:[^\]]*\]\][\s]*'
- % '|'.join(site.validLanguageLinks() + site.family.obsolete.keys())),
-
- }
-
- # if we got a string, compile it as a regular expression
- if type(old) is str or type(old) is unicode:
- if caseInsensitive:
- old = re.compile(old, re.IGNORECASE | re.UNICODE)
- else:
- old = re.compile(old)
-
- dontTouchRegexes = []
- for exc in exceptions:
- if isinstance(exc, str) or isinstance(exc, unicode):
- # assume it's a reference to the exceptionRegexes dictionary
- # defined above.
- if not exceptionRegexes.has_key(exc):
- raise ValueError("Unknown tag type: " + exc)
- dontTouchRegexes.append(exceptionRegexes[exc])
- else:
- # assume it's a regular expression
- dontTouchRegexes.append(exc)
- index = 0
- markerpos = len(text)
- while True:
- match = old.search(text, index)
- if not match:
- # nothing left to replace
- break
-
- # check which exception will occur next.
- nextExceptionMatch = None
- for dontTouchR in dontTouchRegexes:
- excMatch = dontTouchR.search(text, index)
- if excMatch and (
- nextExceptionMatch is None or
- excMatch.start() < nextExceptionMatch.start()):
- nextExceptionMatch = excMatch
-
- if nextExceptionMatch is not None and nextExceptionMatch.start() <= match.start():
- # an HTML comment or text in nowiki tags stands before the next valid match. Skip.
- index = nextExceptionMatch.end()
- else:
- # We found a valid match. Replace it.
- if callable(new):
- # the parameter new can be a function which takes the match as a parameter.
- replacement = new(match)
- else:
- # it is not a function, but a string.
-
- # it is a little hack to make \n work. It would be better to fix it
- # previously, but better than nothing.
- new = new.replace('\\n', '\n')
-
- # We cannot just insert the new string, as it may contain regex
- # group references such as \2 or \g<name>.
- # On the other hand, this approach does not work because it can't
- # handle lookahead or lookbehind (see bug #1731008):
- #replacement = old.sub(new, text[match.start():match.end()])
- #text = text[:match.start()] + replacement + text[match.end():]
-
- # So we have to process the group references manually.
- replacement = new
-
- groupR = re.compile(r'\\(?P<number>\d+)|\\g<(?P<name>.+?)>')
- while True:
- groupMatch = groupR.search(replacement)
- if not groupMatch:
- break
- groupID = groupMatch.group('name') or int(groupMatch.group('number'))
- replacement = replacement[:groupMatch.start()] + match.group(groupID) + replacement[groupMatch.end():]
- text = text[:match.start()] + replacement + text[match.end():]
-
- # continue the search on the remaining text
- if allowoverlap:
- index = match.start() + 1
- else:
- index = match.start() + len(replacement)
- markerpos = match.start() + len(replacement)
- text = text[:markerpos] + marker + text[markerpos:]
- return text
-
-
-def removeDisabledParts(text, tags = ['*']):
- """
- Return text without portions where wiki markup is disabled
-
- Parts that can/will be removed are --
- * HTML comments
- * nowiki tags
- * pre tags
- * includeonly tags
-
- The exact set of parts which should be removed can be passed as the
- 'parts' parameter, which defaults to all.
- """
- regexes = {
- 'comments' : r'<!--.*?-->',
- 'includeonly': r'<includeonly>.*?</includeonly>',
- 'nowiki': r'<nowiki>.*?</nowiki>',
- 'pre': r'<pre>.*?</pre>',
- 'source': r'<source .*?</source>',
- }
- if '*' in tags:
- tags = regexes.keys()
- toRemoveR = re.compile('|'.join([regexes[tag] for tag in tags]),
- re.IGNORECASE | re.DOTALL)
- return toRemoveR.sub('', text)
-
-
-def isDisabled(text, index, tags = ['*']):
- """
- Return True if text[index] is disabled, e.g. by a comment or by nowiki tags.
-
- For the tags parameter, see removeDisabledParts() above.
- """
- # Find a marker that is not already in the text.
- marker = '@@'
- while marker in text:
- marker += '@'
- text = text[:index] + marker + text[index:]
- text = removeDisabledParts(text, tags)
- return (marker not in text)
-
-
-# Functions dealing with interwiki language links
-
-# Note - MediaWiki supports two kinds of interwiki links; interlanguage and
-# interproject. These functions only deal with links to a
-# corresponding page in another language on the same project (e.g.,
-# Wikipedia, Wiktionary, etc.) in another language. They do not find
-# or change links to a different project, or any that are formatted
-# as in-line interwiki links (e.g., "[[:es:Articulo]]". (CONFIRM)
-
-def getLanguageLinks(text, insite = None, pageLink = "[[]]"):
- """
- Return a dict of interlanguage links found in text.
-
- Dict uses language codes as keys and Page objects as values.
- Do not call this routine directly, use Page.interwiki() method
- instead.
-
- """
- if insite == None:
- insite = pywikibot.getSite()
- result = {}
- # Ignore interwiki links within nowiki tags, includeonly tags, pre tags,
- # and HTML comments
- text = removeDisabledParts(text)
-
- # This regular expression will find every link that is possibly an
- # interwiki link.
- # NOTE: language codes are case-insensitive and only consist of basic latin
- # letters and hyphens.
- interwikiR = re.compile(r'\[\[([a-zA-Z\-]+)\s?:([^\[\]\n]*)\]\]')
- for lang, pagetitle in interwikiR.findall(text):
- lang = lang.lower()
- # Check if it really is in fact an interwiki link to a known
- # language, or if it's e.g. a category tag or an internal link
- if lang in insite.family.obsolete:
- lang = insite.family.obsolete[lang]
- if lang in insite.validLanguageLinks():
- if '|' in pagetitle:
- # ignore text after the pipe
- pagetitle = pagetitle[:pagetitle.index('|')]
- # we want the actual page objects rather than the titles
- site = insite.getSite(code = lang)
- try:
- result[site] = pywikibot.Page(site, pagetitle, insite = insite)
- except InvalidTitle:
- output(
- u"[getLanguageLinks] Text contains invalid interwiki link [[%s:%s]]."
- % (lang, pagetitle))
- continue
- return result
-
-
-def removeLanguageLinks(text, site = None, marker = ''):
- """Return text with all interlanguage links removed.
-
- If a link to an unknown language is encountered, a warning is printed.
- If a marker is defined, that string is placed at the location of the
- last occurence of an interwiki link (at the end if there are no
- interwiki links).
-
- """
- if site == None:
- site = pywikibot.getSite()
- if not site.validLanguageLinks():
- return text
- # This regular expression will find every interwiki link, plus trailing
- # whitespace.
- languages = '|'.join(site.validLanguageLinks() + site.family.obsolete.keys())
- interwikiR = re.compile(r'\[\[(%s)\s?:[^\]]*\]\][\s]*'
- % languages, re.IGNORECASE)
- text = replaceExcept(text, interwikiR, '',
- ['nowiki', 'comment', 'math', 'pre', 'source'], marker=marker)
- return text.strip()
-
-
-def replaceLanguageLinks(oldtext, new, site = None):
- """Replace interlanguage links in the text with a new set of links.
-
- 'new' should be a dict with the Site objects as keys, and Page objects
- as values (i.e., just like the dict returned by getLanguageLinks
- function).
-
- """
- # Find a marker that is not already in the text.
- marker = '@@'
- while marker in oldtext:
- marker += '@'
- if site == None:
- site = pywikibot.getSite()
- s = interwikiFormat(new, insite = site)
- s2 = removeLanguageLinks(oldtext, site = site, marker = marker)
- if s:
- if site.language() in site.family.interwiki_attop:
- newtext = s + site.family.interwiki_text_separator + s2.replace(marker,'').strip()
- else:
- # calculate what was after the language links on the page
- firstafter = s2.find(marker) + len(marker)
- # Is there any text in the 'after' part that means we should keep it after?
- if "</noinclude>" in s2[firstafter:]:
- newtext = s2[:firstafter] + s + s2[firstafter:]
- elif site.language() in site.family.categories_last:
- cats = getCategoryLinks(s2, site = site)
- s2 = removeCategoryLinks(s2.replace(marker,'').strip(), site) + site.family.interwiki_text_separator + s
- newtext = replaceCategoryLinks(s2, cats, site=site)
- else:
- newtext = s2.replace(marker,'').strip() + site.family.interwiki_text_separator + s
- newtext = newtext.replace(marker,'')
- else:
- newtext = s2.replace(marker,'')
- return newtext
-
-
-def interwikiFormat(links, insite = None):
- """Convert interwiki link dict into a wikitext string.
-
- 'links' should be a dict with the Site objects as keys, and Page
- objects as values.
-
- Return a unicode string that is formatted for inclusion in insite
- (defaulting to the current site).
- """
- if insite is None:
- insite = pywikibot.getSite()
- if not links:
- return ''
-
- ar = interwikiSort(links.keys(), insite)
- s = []
- for site in ar:
- try:
- link = links[site].aslink(forceInterwiki=True)
- s.append(link)
- except AttributeError:
- s.append(pywikibot.getSite(site).linkto(links[site],
- othersite=insite))
- if insite.lang in insite.family.interwiki_on_one_line:
- sep = u' '
- else:
- sep = u'\r\n'
- s=sep.join(s) + u'\r\n'
- return s
-
-
-# Sort sites according to local interwiki sort logic
-def interwikiSort(sites, insite = None):
- if insite is None:
- insite = pywikibot.getSite()
- if not sites:
- return []
-
- sites.sort()
- putfirst = insite.interwiki_putfirst()
- if putfirst:
- #In this case I might have to change the order
- firstsites = []
- for code in putfirst:
- # The code may not exist in this family?
- if code in insite.family.obsolete:
- code = insite.family.obsolete[code]
- if code in insite.validLanguageLinks():
- site = insite.getSite(code = code)
- if site in sites:
- del sites[sites.index(site)]
- firstsites = firstsites + [site]
- sites = firstsites + sites
- if insite.interwiki_putfirst_doubled(sites): #some implementations return False
- sites = insite.interwiki_putfirst_doubled(sites) + sites
- return sites
-
-
-# Functions dealing with category links
-
-def getCategoryLinks(text, site):
- """Return a list of category links found in text.
-
- List contains Category objects.
- Do not call this routine directly, use Page.categories() instead.
-
- """
- result = []
- # Ignore category links within nowiki tags, pre tags, includeonly tags,
- # and HTML comments
- text = removeDisabledParts(text)
- catNamespace = '|'.join(site.category_namespaces())
- R = re.compile(r'\[\[\s*(?P<namespace>%s)\s*:\s*(?P<catName>.+?)'
- r'(?:\|(?P<sortKey>.+?))?\s*\]\]'
- % catNamespace, re.I)
- for match in R.finditer(text):
- cat = pywikibot.Category(site,
- '%s:%s' % (match.group('namespace'),
- match.group('catName')),
- sortKey = match.group('sortKey'))
- result.append(cat)
- return result
-
-
-def removeCategoryLinks(text, site, marker = ''):
- """Return text with all category links removed.
-
- Put the string marker after the last replacement (at the end of the text
- if there is no replacement).
-
- """
- # This regular expression will find every link that is possibly an
- # interwiki link, plus trailing whitespace. The language code is grouped.
- # NOTE: This assumes that language codes only consist of non-capital
- # ASCII letters and hyphens.
- catNamespace = '|'.join(site.category_namespaces())
- categoryR = re.compile(r'\[\[\s*(%s)\s*:.*?\]\]\s*' % catNamespace, re.I)
- text = replaceExcept(text, categoryR, '', ['nowiki', 'comment', 'math', 'pre', 'source'], marker = marker)
- if marker:
- #avoid having multiple linefeeds at the end of the text
- text = re.sub('\s*%s' % re.escape(marker), '\r\n' + marker, text.strip())
- return text.strip()
-
-
-def replaceCategoryInPlace(oldtext, oldcat, newcat, site=None):
- """Replace the category oldcat with the category newcat and return
- the modified text.
-
- """
- if site is None:
- site = pywikibot.getSite()
-
- catNamespace = '|'.join(site.category_namespaces())
- title = oldcat.titleWithoutNamespace()
- if not title:
- return
- # title might contain regex special characters
- title = re.escape(title)
- # title might not be capitalized correctly on the wiki
- if title[0].isalpha() and not site.nocapitalize:
- title = "[%s%s]" % (title[0].upper(), title[0].lower()) + title[1:]
- # spaces and underscores in page titles are interchangeable, and collapsible
- title = title.replace(r"\ ", "[ _]+").replace(r"\_", "[ _]+")
- categoryR = re.compile(r'\[\[\s*(%s)\s*:\s*%s\s*((?:\|[^]]+)?\]\])'
- % (catNamespace, title), re.I)
- if newcat is None:
- text = replaceExcept(oldtext, categoryR, '',
- ['nowiki', 'comment', 'math', 'pre', 'source'])
- else:
- text = replaceExcept(oldtext, categoryR,
- '[[%s:%s\\2' % (site.namespace(14),
- newcat.titleWithoutNamespace()),
- ['nowiki', 'comment', 'math', 'pre', 'source'])
- return text
-
-
-def replaceCategoryLinks(oldtext, new, site = None, addOnly = False):
- """Replace the category links given in the wikitext given
- in oldtext by the new links given in new.
-
- 'new' should be a list of Category objects.
-
- If addOnly is True, the old category won't be deleted and
- the category(s) given will be added
- (and so they won't replace anything).
- """
-
- # Find a marker that is not already in the text.
- marker = '@@'
- while marker in oldtext:
- marker += '@'
-
- if site is None:
- site = pywikibot.getSite()
- if site.sitename() == 'wikipedia:de' and "{{Personendaten" in oldtext:
- raise Error('The PyWikipediaBot is no longer allowed to touch categories on the German Wikipedia on pages that contain the person data template because of the non-standard placement of that template. See http://de.wikipedia.org/wiki/Hilfe_Diskussion:Personendaten/Archiv/bis_2006…')
-
- s = categoryFormat(new, insite = site)
- if addOnly:
- s2 = oldtext
- else:
- s2 = removeCategoryLinks(oldtext, site = site, marker = marker)
-
- if s:
- if site.language() in site.family.category_attop:
- newtext = s + site.family.category_text_separator + s2
- else:
- # calculate what was after the categories links on the page
- firstafter = s2.find(marker)
- # Is there any text in the 'after' part that means we should keep it after?
- if "</noinclude>" in s2[firstafter:]:
- newtext = s2[:firstafter] + s + s2[firstafter:]
- elif site.language() in site.family.categories_last:
- newtext = s2.replace(marker,'').strip() + site.family.category_text_separator + s
- else:
- interwiki = getLanguageLinks(s2)
- s2 = removeLanguageLinks(s2.replace(marker,''), site) + site.family.category_text_separator + s
- newtext = replaceLanguageLinks(s2, interwiki, site)
- newtext = newtext.replace(marker,'')
- else:
- s2 = s2.replace(marker,'')
- return s2
- return newtext.strip()
-
-
-def categoryFormat(categories, insite = None):
- """Return a string containing links to all categories in a list.
-
- 'categories' should be a list of Category objects.
-
- The string is formatted for inclusion in insite.
-
- """
- if not categories:
- return ''
- if insite is None:
- insite = pywikibot.getSite()
- catLinks = [category.aslink(noInterwiki = True) for category in categories]
- if insite.category_on_one_line():
- sep = ' '
- else:
- sep = '\r\n'
- # Some people don't like the categories sorted
- #catLinks.sort()
- return sep.join(catLinks) + '\r\n'
-
-
-def compileLinkR(withoutBracketed=False, onlyBracketed=False):
- """Return a regex that matches external links."""
- # RFC 2396 says that URLs may only contain certain characters.
- # For this regex we also accept non-allowed characters, so that the bot
- # will later show these links as broken ('Non-ASCII Characters in URL').
- # Note: While allowing parenthesis inside URLs, MediaWiki will regard
- # right parenthesis at the end of the URL as not part of that URL.
- # The same applies to dot, comma, colon and some other characters.
- notAtEnd = '\]\s\)\.:;,<>"'
- # So characters inside the URL can be anything except whitespace,
- # closing squared brackets, quotation marks, greater than and less
- # than, and the last character also can't be parenthesis or another
- # character disallowed by MediaWiki.
- notInside = '\]\s<>"'
- # The first half of this regular expression is required because '' is
- # not allowed inside links. For example, in this wiki text:
- # ''Please see http://www.example.org.''
- # .'' shouldn't be considered as part of the link.
- regex = r'(?P<url>http[s]?://[^' + notInside + ']*?[^' + notAtEnd + '](?=[' + notAtEnd+ ']*\'\')|http[s]?://[^' + notInside + ']*[^' + notAtEnd + '])'
-
- if withoutBracketed:
- regex = r'(?<!\[)' + regex
- elif onlyBracketed:
- regex = r'\[' + regex
- linkR = re.compile(regex)
- return linkR
-
-def extract_templates_and_params(text, get_redirect=False):
- """Return list of template calls found in text.
-
- Return value is a list of tuples. There is one tuple for each use of a
- template in the page, with the template title as the first entry and a
- dict of parameters as the second entry. Parameters are indexed by
- strings; as in MediaWiki, an unnamed parameter is given a parameter name
- with an integer value corresponding to its position among the unnnamed
- parameters, and if this results multiple parameters with the same name
- only the last value provided will be returned.
-
- """
- # remove commented-out stuff etc.
- thistxt = removeDisabledParts(text)
-
- # marker for inside templates or parameters
- marker = u'@@'
- while marker in thistxt:
- marker += u'@'
-
- # marker for links
- marker2 = u'##'
- while marker2 in thistxt:
- marker2 += u'#'
-
- # marker for math
- marker3 = u'%%'
- while marker2 in thistxt:
- marker3 += u'%'
-
- result = []
- inside = {}
- count = 0
- Rtemplate = re.compile(
- ur'{{(msg:)?(?P<name>[^{\|]+?)(\|(?P<params>[^{]+?))?}}')
- Rmath = re.compile(ur'<math>[^<]+</math>')
- Rmarker = re.compile(ur'%s(\d+)%s' % (marker, marker))
- Rmarker2 = re.compile(ur'%s(\d+)%s' % (marker2, marker2))
- Rmarker3 = re.compile(ur'%s(\d+)%s' % (marker3, marker3))
-
- # Replace math with markers
- maths = {}
- count = 0
- for m in Rmath.finditer(thistxt):
- count += 1
- text = m.group()
- thistxt = thistxt.replace(text, '%s%d%s' % (marker3, count, marker3))
- maths[count] = text
-
- while Rtemplate.search(thistxt) is not None:
- for m in Rtemplate.finditer(thistxt):
- # Make sure it is not detected again
- count += 1
- text = m.group()
- thistxt = thistxt.replace(text,
- '%s%d%s' % (marker, count, marker))
- # Make sure stored templates don't contain markers
- for m2 in Rmarker.finditer(text):
- text = text.replace(m2.group(), inside[int(m2.group(1))])
- for m2 in Rmarker3.finditer(text):
- text = text.replace(m2.group(), maths[int(m2.group(1))])
- inside[count] = text
-
- # Name
- name = m.group('name').strip()
- m2 = Rmarker.search(name) or Rmath.search(name)
- if m2 is not None:
- # Doesn't detect templates whose name changes,
- # or templates whose name contains math tags
- continue
- # Parameters
- paramString = m.group('params')
- params = {}
- numbered_param = 1
- if paramString:
- # Replace wikilinks with markers
- links = {}
- count2 = 0
- for m2 in pywikibot.link_regex.finditer(paramString):
- count2 += 1
- text = m2.group(0)
- paramString = paramString.replace(text,
- '%s%d%s' % (marker2, count2, marker2))
- links[count2] = text
- # Parse string
- markedParams = paramString.split('|')
- # Replace markers
- for param in markedParams:
- if "=" in param:
- param_name, param_val = param.split("=", 1)
- else:
- param_name = unicode(numbered_param)
- param_val = param
- numbered_param += 1
- for m2 in Rmarker.finditer(param_val):
- param_val = param_val.replace(m2.group(),
- inside[int(m2.group(1))])
- for m2 in Rmarker2.finditer(param_val):
- param_val = param_val.replace(m2.group(),
- links[int(m2.group(1))])
- for m2 in Rmarker3.finditer(param_val):
- param_val = param_val.replace(m2.group(),
- maths[int(m2.group(1))])
- params[param_name] = param_val
-
- # Add it to the result
- result.append((name, params))
- return result
-
+# -*- coding: utf-8 -*-
+"""
+Functions for manipulating wiki-text.
+
+Unless otherwise noted, all functions take a unicode string as the argument
+and return a unicode string.
+
+"""
+#
+# (C) Pywikipedia bot team, 2008
+#
+# Distributed under the terms of the MIT license.
+#
+__version__ = '$Id$'
+
+
+import pywikibot
+import re
+
+
+def unescape(s):
+ """Replace escaped HTML-special characters by their originals"""
+ if '&' not in s:
+ return s
+ s = s.replace("<", "<")
+ s = s.replace(">", ">")
+ s = s.replace("'", "'")
+ s = s.replace(""", '"')
+ s = s.replace("&", "&") # Must be last
+ return s
+
+
+def replaceExcept(text, old, new, exceptions, caseInsensitive=False,
+ allowoverlap=False, marker = '', site = None):
+ """
+ Return text with 'old' replaced by 'new', ignoring specified types of text.
+
+ Skips occurences of 'old' within exceptions; e.g., within nowiki tags or
+ HTML comments. If caseInsensitive is true, then use case insensitive
+ regex matching. If allowoverlap is true, overlapping occurences are all
+ replaced (watch out when using this, it might lead to infinite loops!).
+
+ Parameters:
+ text - a unicode string
+ old - a compiled regular expression
+ new - a unicode string (which can contain regular
+ expression references), or a function which takes
+ a match object as parameter. See parameter repl of
+ re.sub().
+ exceptions - a list of strings which signal what to leave out,
+ e.g. ['math', 'table', 'template']
+ caseInsensitive - a boolean
+ marker - a string that will be added to the last replacement;
+ if nothing is changed, it is added at the end
+
+ """
+ if site is None:
+ site = pywikibot.getSite()
+
+ exceptionRegexes = {
+ 'comment': re.compile(r'(?s)<!--.*?-->'),
+ # section headers
+ 'header': re.compile(r'\r\n=+.+=+ *\r\n'),
+ 'includeonly': re.compile(r'(?is)<includeonly>.*?</includeonly>'),
+ 'math': re.compile(r'(?is)<math>.*?</math>'),
+ 'noinclude': re.compile(r'(?is)<noinclude>.*?</noinclude>'),
+ # wiki tags are ignored inside nowiki tags.
+ 'nowiki': re.compile(r'(?is)<nowiki>.*?</nowiki>'),
+ # preformatted text
+ 'pre': re.compile(r'(?ism)<pre>.*?</pre>'),
+ 'source': re.compile(r'(?is)<source .*?</source>'),
+ # inline references
+ 'ref': re.compile(r'(?ism)<ref[ >].*?</ref>'),
+ 'timeline': re.compile(r'(?is)<timeline>.*?</timeline>'),
+ # lines that start with a space are shown in a monospace font and
+ # have whitespace preserved.
+ 'startspace': re.compile(r'(?m)^ (.*?)$'),
+ # tables often have whitespace that is used to improve wiki
+ # source code readability.
+ # TODO: handle nested tables.
+ 'table': re.compile(r'(?ims)^{\|.*?^\|}|<table>.*?</table>'),
+ # templates with parameters often have whitespace that is used to
+ # improve wiki source code readability.
+ # 'template': re.compile(r'(?s){{.*?}}'),
+ # The regex above fails on nested templates. This regex can handle
+ # templates cascaded up to level 3, but no deeper. For arbitrary
+ # depth, we'd need recursion which can't be done in Python's re.
+ # After all, the language of correct parenthesis words is not regular.
+ 'template': re.compile(r'(?s){{(({{(({{.*?}})|.)*}})|.)*}}'),
+ 'hyperlink': compileLinkR(),
+ 'gallery': re.compile(r'(?is)<gallery.*?>.*?</gallery>'),
+ # this matches internal wikilinks, but also interwiki, categories, and
+ # images.
+ 'link': re.compile(r'\[\[[^\]\|]*(\|[^\]]*)?\]\]'),
+ 'interwiki': re.compile(r'(?i)\[\[(%s)\s?:[^\]]*\]\][\s]*'
+ % '|'.join(site.validLanguageLinks() + site.family.obsolete.keys())),
+
+ }
+
+ # if we got a string, compile it as a regular expression
+ if type(old) is str or type(old) is unicode:
+ if caseInsensitive:
+ old = re.compile(old, re.IGNORECASE | re.UNICODE)
+ else:
+ old = re.compile(old)
+
+ dontTouchRegexes = []
+ for exc in exceptions:
+ if isinstance(exc, str) or isinstance(exc, unicode):
+ # assume it's a reference to the exceptionRegexes dictionary
+ # defined above.
+ if not exceptionRegexes.has_key(exc):
+ raise ValueError("Unknown tag type: " + exc)
+ dontTouchRegexes.append(exceptionRegexes[exc])
+ else:
+ # assume it's a regular expression
+ dontTouchRegexes.append(exc)
+ index = 0
+ markerpos = len(text)
+ while True:
+ match = old.search(text, index)
+ if not match:
+ # nothing left to replace
+ break
+
+ # check which exception will occur next.
+ nextExceptionMatch = None
+ for dontTouchR in dontTouchRegexes:
+ excMatch = dontTouchR.search(text, index)
+ if excMatch and (
+ nextExceptionMatch is None or
+ excMatch.start() < nextExceptionMatch.start()):
+ nextExceptionMatch = excMatch
+
+ if nextExceptionMatch is not None and nextExceptionMatch.start() <= match.start():
+ # an HTML comment or text in nowiki tags stands before the next valid match. Skip.
+ index = nextExceptionMatch.end()
+ else:
+ # We found a valid match. Replace it.
+ if callable(new):
+ # the parameter new can be a function which takes the match as a parameter.
+ replacement = new(match)
+ else:
+ # it is not a function, but a string.
+
+ # it is a little hack to make \n work. It would be better to fix it
+ # previously, but better than nothing.
+ new = new.replace('\\n', '\n')
+
+ # We cannot just insert the new string, as it may contain regex
+ # group references such as \2 or \g<name>.
+ # On the other hand, this approach does not work because it can't
+ # handle lookahead or lookbehind (see bug #1731008):
+ #replacement = old.sub(new, text[match.start():match.end()])
+ #text = text[:match.start()] + replacement + text[match.end():]
+
+ # So we have to process the group references manually.
+ replacement = new
+
+ groupR = re.compile(r'\\(?P<number>\d+)|\\g<(?P<name>.+?)>')
+ while True:
+ groupMatch = groupR.search(replacement)
+ if not groupMatch:
+ break
+ groupID = groupMatch.group('name') or int(groupMatch.group('number'))
+ replacement = replacement[:groupMatch.start()] + match.group(groupID) + replacement[groupMatch.end():]
+ text = text[:match.start()] + replacement + text[match.end():]
+
+ # continue the search on the remaining text
+ if allowoverlap:
+ index = match.start() + 1
+ else:
+ index = match.start() + len(replacement)
+ markerpos = match.start() + len(replacement)
+ text = text[:markerpos] + marker + text[markerpos:]
+ return text
+
+
+def removeDisabledParts(text, tags = ['*']):
+ """
+ Return text without portions where wiki markup is disabled
+
+ Parts that can/will be removed are --
+ * HTML comments
+ * nowiki tags
+ * pre tags
+ * includeonly tags
+
+ The exact set of parts which should be removed can be passed as the
+ 'parts' parameter, which defaults to all.
+ """
+ regexes = {
+ 'comments' : r'<!--.*?-->',
+ 'includeonly': r'<includeonly>.*?</includeonly>',
+ 'nowiki': r'<nowiki>.*?</nowiki>',
+ 'pre': r'<pre>.*?</pre>',
+ 'source': r'<source .*?</source>',
+ }
+ if '*' in tags:
+ tags = regexes.keys()
+ toRemoveR = re.compile('|'.join([regexes[tag] for tag in tags]),
+ re.IGNORECASE | re.DOTALL)
+ return toRemoveR.sub('', text)
+
+
+def isDisabled(text, index, tags = ['*']):
+ """
+ Return True if text[index] is disabled, e.g. by a comment or by nowiki tags.
+
+ For the tags parameter, see removeDisabledParts() above.
+ """
+ # Find a marker that is not already in the text.
+ marker = '@@'
+ while marker in text:
+ marker += '@'
+ text = text[:index] + marker + text[index:]
+ text = removeDisabledParts(text, tags)
+ return (marker not in text)
+
+
+# Functions dealing with interwiki language links
+
+# Note - MediaWiki supports two kinds of interwiki links; interlanguage and
+# interproject. These functions only deal with links to a
+# corresponding page in another language on the same project (e.g.,
+# Wikipedia, Wiktionary, etc.) in another language. They do not find
+# or change links to a different project, or any that are formatted
+# as in-line interwiki links (e.g., "[[:es:Articulo]]". (CONFIRM)
+
+def getLanguageLinks(text, insite = None, pageLink = "[[]]"):
+ """
+ Return a dict of interlanguage links found in text.
+
+ Dict uses language codes as keys and Page objects as values.
+ Do not call this routine directly, use Page.interwiki() method
+ instead.
+
+ """
+ if insite == None:
+ insite = pywikibot.getSite()
+ result = {}
+ # Ignore interwiki links within nowiki tags, includeonly tags, pre tags,
+ # and HTML comments
+ text = removeDisabledParts(text)
+
+ # This regular expression will find every link that is possibly an
+ # interwiki link.
+ # NOTE: language codes are case-insensitive and only consist of basic latin
+ # letters and hyphens.
+ interwikiR = re.compile(r'\[\[([a-zA-Z\-]+)\s?:([^\[\]\n]*)\]\]')
+ for lang, pagetitle in interwikiR.findall(text):
+ lang = lang.lower()
+ # Check if it really is in fact an interwiki link to a known
+ # language, or if it's e.g. a category tag or an internal link
+ if lang in insite.family.obsolete:
+ lang = insite.family.obsolete[lang]
+ if lang in insite.validLanguageLinks():
+ if '|' in pagetitle:
+ # ignore text after the pipe
+ pagetitle = pagetitle[:pagetitle.index('|')]
+ # we want the actual page objects rather than the titles
+ site = insite.getSite(code = lang)
+ try:
+ result[site] = pywikibot.Page(site, pagetitle, insite = insite)
+ except InvalidTitle:
+ output(
+ u"[getLanguageLinks] Text contains invalid interwiki link [[%s:%s]]."
+ % (lang, pagetitle))
+ continue
+ return result
+
+
+def removeLanguageLinks(text, site = None, marker = ''):
+ """Return text with all interlanguage links removed.
+
+ If a link to an unknown language is encountered, a warning is printed.
+ If a marker is defined, that string is placed at the location of the
+ last occurence of an interwiki link (at the end if there are no
+ interwiki links).
+
+ """
+ if site == None:
+ site = pywikibot.getSite()
+ if not site.validLanguageLinks():
+ return text
+ # This regular expression will find every interwiki link, plus trailing
+ # whitespace.
+ languages = '|'.join(site.validLanguageLinks() + site.family.obsolete.keys())
+ interwikiR = re.compile(r'\[\[(%s)\s?:[^\]]*\]\][\s]*'
+ % languages, re.IGNORECASE)
+ text = replaceExcept(text, interwikiR, '',
+ ['nowiki', 'comment', 'math', 'pre', 'source'], marker=marker)
+ return text.strip()
+
+
+def replaceLanguageLinks(oldtext, new, site = None):
+ """Replace interlanguage links in the text with a new set of links.
+
+ 'new' should be a dict with the Site objects as keys, and Page objects
+ as values (i.e., just like the dict returned by getLanguageLinks
+ function).
+
+ """
+ # Find a marker that is not already in the text.
+ marker = '@@'
+ while marker in oldtext:
+ marker += '@'
+ if site == None:
+ site = pywikibot.getSite()
+ s = interwikiFormat(new, insite = site)
+ s2 = removeLanguageLinks(oldtext, site = site, marker = marker)
+ if s:
+ if site.language() in site.family.interwiki_attop:
+ newtext = s + site.family.interwiki_text_separator + s2.replace(marker,'').strip()
+ else:
+ # calculate what was after the language links on the page
+ firstafter = s2.find(marker) + len(marker)
+ # Is there any text in the 'after' part that means we should keep it after?
+ if "</noinclude>" in s2[firstafter:]:
+ newtext = s2[:firstafter] + s + s2[firstafter:]
+ elif site.language() in site.family.categories_last:
+ cats = getCategoryLinks(s2, site = site)
+ s2 = removeCategoryLinks(s2.replace(marker,'').strip(), site) + site.family.interwiki_text_separator + s
+ newtext = replaceCategoryLinks(s2, cats, site=site)
+ else:
+ newtext = s2.replace(marker,'').strip() + site.family.interwiki_text_separator + s
+ newtext = newtext.replace(marker,'')
+ else:
+ newtext = s2.replace(marker,'')
+ return newtext
+
+
+def interwikiFormat(links, insite = None):
+ """Convert interwiki link dict into a wikitext string.
+
+ 'links' should be a dict with the Site objects as keys, and Page
+ objects as values.
+
+ Return a unicode string that is formatted for inclusion in insite
+ (defaulting to the current site).
+ """
+ if insite is None:
+ insite = pywikibot.getSite()
+ if not links:
+ return ''
+
+ ar = interwikiSort(links.keys(), insite)
+ s = []
+ for site in ar:
+ try:
+ link = links[site].aslink(forceInterwiki=True)
+ s.append(link)
+ except AttributeError:
+ s.append(pywikibot.getSite(site).linkto(links[site],
+ othersite=insite))
+ if insite.lang in insite.family.interwiki_on_one_line:
+ sep = u' '
+ else:
+ sep = u'\r\n'
+ s=sep.join(s) + u'\r\n'
+ return s
+
+
+# Sort sites according to local interwiki sort logic
+def interwikiSort(sites, insite = None):
+ if insite is None:
+ insite = pywikibot.getSite()
+ if not sites:
+ return []
+
+ sites.sort()
+ putfirst = insite.interwiki_putfirst()
+ if putfirst:
+ #In this case I might have to change the order
+ firstsites = []
+ for code in putfirst:
+ # The code may not exist in this family?
+ if code in insite.family.obsolete:
+ code = insite.family.obsolete[code]
+ if code in insite.validLanguageLinks():
+ site = insite.getSite(code = code)
+ if site in sites:
+ del sites[sites.index(site)]
+ firstsites = firstsites + [site]
+ sites = firstsites + sites
+ if insite.interwiki_putfirst_doubled(sites): #some implementations return False
+ sites = insite.interwiki_putfirst_doubled(sites) + sites
+ return sites
+
+
+# Functions dealing with category links
+
+def getCategoryLinks(text, site):
+ """Return a list of category links found in text.
+
+ List contains Category objects.
+ Do not call this routine directly, use Page.categories() instead.
+
+ """
+ result = []
+ # Ignore category links within nowiki tags, pre tags, includeonly tags,
+ # and HTML comments
+ text = removeDisabledParts(text)
+ catNamespace = '|'.join(site.category_namespaces())
+ R = re.compile(r'\[\[\s*(?P<namespace>%s)\s*:\s*(?P<catName>.+?)'
+ r'(?:\|(?P<sortKey>.+?))?\s*\]\]'
+ % catNamespace, re.I)
+ for match in R.finditer(text):
+ cat = pywikibot.Category(site,
+ '%s:%s' % (match.group('namespace'),
+ match.group('catName')),
+ sortKey = match.group('sortKey'))
+ result.append(cat)
+ return result
+
+
+def removeCategoryLinks(text, site, marker = ''):
+ """Return text with all category links removed.
+
+ Put the string marker after the last replacement (at the end of the text
+ if there is no replacement).
+
+ """
+ # This regular expression will find every link that is possibly an
+ # interwiki link, plus trailing whitespace. The language code is grouped.
+ # NOTE: This assumes that language codes only consist of non-capital
+ # ASCII letters and hyphens.
+ catNamespace = '|'.join(site.category_namespaces())
+ categoryR = re.compile(r'\[\[\s*(%s)\s*:.*?\]\]\s*' % catNamespace, re.I)
+ text = replaceExcept(text, categoryR, '', ['nowiki', 'comment', 'math', 'pre', 'source'], marker = marker)
+ if marker:
+ #avoid having multiple linefeeds at the end of the text
+ text = re.sub('\s*%s' % re.escape(marker), '\r\n' + marker, text.strip())
+ return text.strip()
+
+
+def replaceCategoryInPlace(oldtext, oldcat, newcat, site=None):
+ """Replace the category oldcat with the category newcat and return
+ the modified text.
+
+ """
+ if site is None:
+ site = pywikibot.getSite()
+
+ catNamespace = '|'.join(site.category_namespaces())
+ title = oldcat.titleWithoutNamespace()
+ if not title:
+ return
+ # title might contain regex special characters
+ title = re.escape(title)
+ # title might not be capitalized correctly on the wiki
+ if title[0].isalpha() and not site.nocapitalize:
+ title = "[%s%s]" % (title[0].upper(), title[0].lower()) + title[1:]
+ # spaces and underscores in page titles are interchangeable, and collapsible
+ title = title.replace(r"\ ", "[ _]+").replace(r"\_", "[ _]+")
+ categoryR = re.compile(r'\[\[\s*(%s)\s*:\s*%s\s*((?:\|[^]]+)?\]\])'
+ % (catNamespace, title), re.I)
+ if newcat is None:
+ text = replaceExcept(oldtext, categoryR, '',
+ ['nowiki', 'comment', 'math', 'pre', 'source'])
+ else:
+ text = replaceExcept(oldtext, categoryR,
+ '[[%s:%s\\2' % (site.namespace(14),
+ newcat.titleWithoutNamespace()),
+ ['nowiki', 'comment', 'math', 'pre', 'source'])
+ return text
+
+
+def replaceCategoryLinks(oldtext, new, site = None, addOnly = False):
+ """Replace the category links given in the wikitext given
+ in oldtext by the new links given in new.
+
+ 'new' should be a list of Category objects.
+
+ If addOnly is True, the old category won't be deleted and
+ the category(s) given will be added
+ (and so they won't replace anything).
+ """
+
+ # Find a marker that is not already in the text.
+ marker = '@@'
+ while marker in oldtext:
+ marker += '@'
+
+ if site is None:
+ site = pywikibot.getSite()
+ if site.sitename() == 'wikipedia:de' and "{{Personendaten" in oldtext:
+ raise Error('The PyWikipediaBot is no longer allowed to touch categories on the German Wikipedia on pages that contain the person data template because of the non-standard placement of that template. See http://de.wikipedia.org/wiki/Hilfe_Diskussion:Personendaten/Archiv/bis_2006…')
+
+ s = categoryFormat(new, insite = site)
+ if addOnly:
+ s2 = oldtext
+ else:
+ s2 = removeCategoryLinks(oldtext, site = site, marker = marker)
+
+ if s:
+ if site.language() in site.family.category_attop:
+ newtext = s + site.family.category_text_separator + s2
+ else:
+ # calculate what was after the categories links on the page
+ firstafter = s2.find(marker)
+ # Is there any text in the 'after' part that means we should keep it after?
+ if "</noinclude>" in s2[firstafter:]:
+ newtext = s2[:firstafter] + s + s2[firstafter:]
+ elif site.language() in site.family.categories_last:
+ newtext = s2.replace(marker,'').strip() + site.family.category_text_separator + s
+ else:
+ interwiki = getLanguageLinks(s2)
+ s2 = removeLanguageLinks(s2.replace(marker,''), site) + site.family.category_text_separator + s
+ newtext = replaceLanguageLinks(s2, interwiki, site)
+ newtext = newtext.replace(marker,'')
+ else:
+ s2 = s2.replace(marker,'')
+ return s2
+ return newtext.strip()
+
+
+def categoryFormat(categories, insite = None):
+ """Return a string containing links to all categories in a list.
+
+ 'categories' should be a list of Category objects.
+
+ The string is formatted for inclusion in insite.
+
+ """
+ if not categories:
+ return ''
+ if insite is None:
+ insite = pywikibot.getSite()
+ catLinks = [category.aslink(noInterwiki = True) for category in categories]
+ if insite.category_on_one_line():
+ sep = ' '
+ else:
+ sep = '\r\n'
+ # Some people don't like the categories sorted
+ #catLinks.sort()
+ return sep.join(catLinks) + '\r\n'
+
+
+def compileLinkR(withoutBracketed=False, onlyBracketed=False):
+ """Return a regex that matches external links."""
+ # RFC 2396 says that URLs may only contain certain characters.
+ # For this regex we also accept non-allowed characters, so that the bot
+ # will later show these links as broken ('Non-ASCII Characters in URL').
+ # Note: While allowing parenthesis inside URLs, MediaWiki will regard
+ # right parenthesis at the end of the URL as not part of that URL.
+ # The same applies to dot, comma, colon and some other characters.
+ notAtEnd = '\]\s\)\.:;,<>"'
+ # So characters inside the URL can be anything except whitespace,
+ # closing squared brackets, quotation marks, greater than and less
+ # than, and the last character also can't be parenthesis or another
+ # character disallowed by MediaWiki.
+ notInside = '\]\s<>"'
+ # The first half of this regular expression is required because '' is
+ # not allowed inside links. For example, in this wiki text:
+ # ''Please see http://www.example.org.''
+ # .'' shouldn't be considered as part of the link.
+ regex = r'(?P<url>http[s]?://[^' + notInside + ']*?[^' + notAtEnd + '](?=[' + notAtEnd+ ']*\'\')|http[s]?://[^' + notInside + ']*[^' + notAtEnd + '])'
+
+ if withoutBracketed:
+ regex = r'(?<!\[)' + regex
+ elif onlyBracketed:
+ regex = r'\[' + regex
+ linkR = re.compile(regex)
+ return linkR
+
+def extract_templates_and_params(text, get_redirect=False):
+ """Return list of template calls found in text.
+
+ Return value is a list of tuples. There is one tuple for each use of a
+ template in the page, with the template title as the first entry and a
+ dict of parameters as the second entry. Parameters are indexed by
+ strings; as in MediaWiki, an unnamed parameter is given a parameter name
+ with an integer value corresponding to its position among the unnnamed
+ parameters, and if this results multiple parameters with the same name
+ only the last value provided will be returned.
+
+ """
+ # remove commented-out stuff etc.
+ thistxt = removeDisabledParts(text)
+
+ # marker for inside templates or parameters
+ marker = u'@@'
+ while marker in thistxt:
+ marker += u'@'
+
+ # marker for links
+ marker2 = u'##'
+ while marker2 in thistxt:
+ marker2 += u'#'
+
+ # marker for math
+ marker3 = u'%%'
+ while marker2 in thistxt:
+ marker3 += u'%'
+
+ result = []
+ inside = {}
+ count = 0
+ Rtemplate = re.compile(
+ ur'{{(msg:)?(?P<name>[^{\|]+?)(\|(?P<params>[^{]+?))?}}')
+ Rmath = re.compile(ur'<math>[^<]+</math>')
+ Rmarker = re.compile(ur'%s(\d+)%s' % (marker, marker))
+ Rmarker2 = re.compile(ur'%s(\d+)%s' % (marker2, marker2))
+ Rmarker3 = re.compile(ur'%s(\d+)%s' % (marker3, marker3))
+
+ # Replace math with markers
+ maths = {}
+ count = 0
+ for m in Rmath.finditer(thistxt):
+ count += 1
+ text = m.group()
+ thistxt = thistxt.replace(text, '%s%d%s' % (marker3, count, marker3))
+ maths[count] = text
+
+ while Rtemplate.search(thistxt) is not None:
+ for m in Rtemplate.finditer(thistxt):
+ # Make sure it is not detected again
+ count += 1
+ text = m.group()
+ thistxt = thistxt.replace(text,
+ '%s%d%s' % (marker, count, marker))
+ # Make sure stored templates don't contain markers
+ for m2 in Rmarker.finditer(text):
+ text = text.replace(m2.group(), inside[int(m2.group(1))])
+ for m2 in Rmarker3.finditer(text):
+ text = text.replace(m2.group(), maths[int(m2.group(1))])
+ inside[count] = text
+
+ # Name
+ name = m.group('name').strip()
+ m2 = Rmarker.search(name) or Rmath.search(name)
+ if m2 is not None:
+ # Doesn't detect templates whose name changes,
+ # or templates whose name contains math tags
+ continue
+ # Parameters
+ paramString = m.group('params')
+ params = {}
+ numbered_param = 1
+ if paramString:
+ # Replace wikilinks with markers
+ links = {}
+ count2 = 0
+ for m2 in pywikibot.link_regex.finditer(paramString):
+ count2 += 1
+ text = m2.group(0)
+ paramString = paramString.replace(text,
+ '%s%d%s' % (marker2, count2, marker2))
+ links[count2] = text
+ # Parse string
+ markedParams = paramString.split('|')
+ # Replace markers
+ for param in markedParams:
+ if "=" in param:
+ param_name, param_val = param.split("=", 1)
+ else:
+ param_name = unicode(numbered_param)
+ param_val = param
+ numbered_param += 1
+ for m2 in Rmarker.finditer(param_val):
+ param_val = param_val.replace(m2.group(),
+ inside[int(m2.group(1))])
+ for m2 in Rmarker2.finditer(param_val):
+ param_val = param_val.replace(m2.group(),
+ links[int(m2.group(1))])
+ for m2 in Rmarker3.finditer(param_val):
+ param_val = param_val.replace(m2.group(),
+ maths[int(m2.group(1))])
+ params[param_name] = param_val
+
+ # Add it to the result
+ result.append((name, params))
+ return result
+
Property changes on: branches/rewrite/pywikibot/textlib.py
___________________________________________________________________
Added: svn:keywords
+ Author Date Id Revision
Added: svn:eol-style
+ native
Modified: branches/rewrite/pywikibot/throttle.py
===================================================================
--- branches/rewrite/pywikibot/throttle.py 2008-12-16 19:34:48 UTC (rev 6155)
+++ branches/rewrite/pywikibot/throttle.py 2008-12-16 19:40:20 UTC (rev 6156)
@@ -1,275 +1,275 @@
-# -*- coding: utf-8 -*-
-"""
-Mechanics to slow down wiki read and/or write rate.
-"""
-#
-# (C) Pywikipedia bot team, 2008
-#
-# Distributed under the terms of the MIT license.
-#
-__version__ = '$Id: $'
-
-import pywikibot
-from pywikibot import config2 as config
-
-import logging
-import math
-import threading
-import time
-
-logger = logging.getLogger("wiki.throttle")
-
-pid = False # global process identifier
- # when the first Throttle is instantiated, it will set this
- # variable to a positive integer, which will apply to all
- # throttle objects created by this process.
-
-
-class Throttle(object):
- """Control rate of access to wiki server
-
- Calling this object blocks the calling thread until at least 'delay'
- seconds have passed since the previous call.
-
- Each Site initiates one Throttle object (site.throttle) to control the
- rate of access.
-
- """
- def __init__(self, site, mindelay=None, maxdelay=None, writedelay=None,
- multiplydelay=True, verbosedelay=False):
- self.lock = threading.RLock()
- self.mysite = str(site)
- self.logfn = config.datafilepath('throttle.log')
- self.mindelay = mindelay
- if self.mindelay is None:
- self.mindelay = config.minthrottle
- self.maxdelay = maxdelay
- if self.maxdelay is None:
- self.maxdelay = config.maxthrottle
- self.writedelay = writedelay
- self.last_read = 0
- self.last_write = 0
- self.next_multiplicity = 1.0
- self.checkdelay = 300 # Check logfile again after this many seconds
- self.dropdelay = 600 # Ignore processes that have not made
- # a check in this many seconds
- self.releasepid = 1200 # Free the process id after this many seconds
- self.lastwait = 0.0
- self.delay = 0
- self.verbosedelay = verbosedelay
- if multiplydelay:
- self.checkMultiplicity()
- self.setDelays()
-
- def checkMultiplicity(self):
- """Count running processes for site and set process_multiplicity."""
- global pid
- self.lock.acquire()
- mysite = self.mysite
- logger.debug("Checking multiplicity: pid = %(pid)s" % globals())
- try:
- processes = []
- my_pid = pid or 1 # start at 1 if global pid not yet set
- count = 1
- # open throttle.log
- try:
- f = open(self.logfn, 'r')
- except IOError:
- if not pid:
- pass
- else:
- raise
- else:
- now = time.time()
- for line in f.readlines():
- # parse line; format is "pid timestamp site"
- try:
- line = line.split(' ')
- this_pid = int(line[0])
- ptime = int(line[1].split('.')[0])
- this_site = line[2].rstrip()
- except (IndexError, ValueError):
- continue # Sometimes the file gets corrupted
- # ignore that line
- if now - ptime > self.releasepid:
- continue # process has expired, drop from file
- if now - ptime <= self.dropdelay \
- and this_site == mysite \
- and this_pid != pid:
- count += 1
- if this_site != self.mysite or this_pid != pid:
- processes.append({'pid': this_pid,
- 'time': ptime,
- 'site': this_site})
- if not pid and this_pid >= my_pid:
- my_pid = this_pid+1 # next unused process id
-
- if not pid:
- pid = my_pid
- self.checktime = time.time()
- processes.append({'pid': pid,
- 'time': self.checktime,
- 'site': mysite})
- f = open(self.logfn, 'w')
- processes.sort(key=lambda p:(p['pid'], p['site']))
- for p in processes:
- f.write("%(pid)s %(time)s %(site)s\n" % p)
- f.close()
- self.process_multiplicity = count
- if self.verbosedelay:
- logger.info(
-u"Found %(count)s %(mysite)s processes running, including this one."
- % locals())
- finally:
- self.lock.release()
-
- def setDelays(self, delay=None, writedelay=None, absolute=False):
- """Set the nominal delays in seconds. Defaults to config values."""
- self.lock.acquire()
- try:
- maxdelay = self.maxdelay
- if delay is None:
- delay = self.mindelay
- if writedelay is None:
- writedelay = config.put_throttle
- if absolute:
- self.maxdelay = delay
- self.mindelay = delay
- self.delay = delay
- self.writedelay = min(max(self.mindelay, writedelay),
- self.maxdelay)
- # Start the delay count now, not at the next check
- self.last_read = self.last_write = time.time()
- finally:
- self.lock.release()
-
- def getDelay(self, write=False):
- """Return the actual delay, accounting for multiple processes.
-
- This value is the maximum wait between reads/writes, not taking
- account of how much time has elapsed since the last access.
-
- """
- global pid
- if write:
- thisdelay = self.writedelay
- else:
- thisdelay = self.delay
- if pid: # If set, we're checking for multiple processes
- if time.time() > self.checktime + self.checkdelay:
- self.checkMultiplicity()
- if thisdelay < (self.mindelay * self.next_multiplicity):
- thisdelay = self.mindelay * self.next_multiplicity
- elif thisdelay > self.maxdelay:
- thisdelay = self.maxdelay
- thisdelay *= self.process_multiplicity
- return thisdelay
-
- def waittime(self, write=False):
- """Return waiting time in seconds if a query would be made right now"""
- # Take the previous requestsize in account calculating the desired
- # delay this time
- thisdelay = self.getDelay(write=write)
- now = time.time()
- if write:
- ago = now - self.last_write
- else:
- ago = now - self.last_read
- if ago < thisdelay:
- delta = thisdelay - ago
- return delta
- else:
- return 0.0
-
- def drop(self):
- """Remove me from the list of running bot processes."""
- # drop all throttles with this process's pid, regardless of site
- self.checktime = 0
- processes = []
- try:
- f = open(self.logfn, 'r')
- except IOError:
- return
- else:
- now = time.time()
- for line in f.readlines():
- try:
- line = line.split(' ')
- this_pid = int(line[0])
- ptime = int(line[1].split('.')[0])
- this_site = line[2].rstrip()
- except (IndexError,ValueError):
- continue # Sometimes the file gets corrupted
- # ignore that line
- if now - ptime <= self.releasepid \
- and this_pid != pid:
- processes.append({'pid': this_pid,
- 'time': ptime,
- 'site': this_site})
- f = open(self.logfn, 'w')
- processes.sort(key=lambda p:p['pid'])
- for p in processes:
- f.write("%(pid)s %(time)s %(site)s\n" % p)
- f.close()
-
- def __call__(self, requestsize=1, write=False):
- """
- Block the calling program if the throttle time has not expired.
-
- Parameter requestsize is the number of Pages to be read/written;
- multiply delay time by an appropriate factor.
-
- Because this seizes the throttle lock, it will prevent any other
- thread from writing to the same site until the wait expires.
-
- """
- self.lock.acquire()
- try:
- wait = self.waittime(write=write)
- # Calculate the multiplicity of the next delay based on how
- # big the request is that is being posted now.
- # We want to add "one delay" for each factor of two in the
- # size of the request. Getting 64 pages at once allows 6 times
- # the delay time for the server.
- self.next_multiplicity = math.log(1+requestsize)/math.log(2.0)
- # Announce the delay if it exceeds a preset limit
- if wait > config.noisysleep:
- logger.info(u"Sleeping for %(wait).1f seconds, %(now)s"
- % {'wait': wait,
- 'now': time.strftime("%Y-%m-%d %H:%M:%S",
- time.localtime())
- } )
- time.sleep(wait)
- if write:
- self.last_write = time.time()
- else:
- self.last_read = time.time()
- finally:
- self.lock.release()
-
- def lag(self, lagtime):
- """
- Seize the throttle lock due to server lag.
-
- This will prevent any thread from accessing this site.
-
- """
- started = time.time()
- self.lock.acquire()
- try:
- # start at 1/2 the current server lag time
- # wait at least 5 seconds but not more than 120 seconds
- delay = min(max(5, lagtime//2), 120)
- # account for any time we waited while acquiring the lock
- wait = delay - (time.time() - started)
- if wait > 0:
- if wait > config.noisysleep:
- logger.info(u"Sleeping for %(wait).1f seconds, %(now)s"
- % {'wait': wait,
- 'now': time.strftime("%Y-%m-%d %H:%M:%S",
- time.localtime())
- } )
- time.sleep(wait)
- finally:
- self.lock.release()
-
+# -*- coding: utf-8 -*-
+"""
+Mechanics to slow down wiki read and/or write rate.
+"""
+#
+# (C) Pywikipedia bot team, 2008
+#
+# Distributed under the terms of the MIT license.
+#
+__version__ = '$Id$'
+
+import pywikibot
+from pywikibot import config2 as config
+
+import logging
+import math
+import threading
+import time
+
+logger = logging.getLogger("wiki.throttle")
+
+pid = False # global process identifier
+ # when the first Throttle is instantiated, it will set this
+ # variable to a positive integer, which will apply to all
+ # throttle objects created by this process.
+
+
+class Throttle(object):
+ """Control rate of access to wiki server
+
+ Calling this object blocks the calling thread until at least 'delay'
+ seconds have passed since the previous call.
+
+ Each Site initiates one Throttle object (site.throttle) to control the
+ rate of access.
+
+ """
+ def __init__(self, site, mindelay=None, maxdelay=None, writedelay=None,
+ multiplydelay=True, verbosedelay=False):
+ self.lock = threading.RLock()
+ self.mysite = str(site)
+ self.logfn = config.datafilepath('throttle.log')
+ self.mindelay = mindelay
+ if self.mindelay is None:
+ self.mindelay = config.minthrottle
+ self.maxdelay = maxdelay
+ if self.maxdelay is None:
+ self.maxdelay = config.maxthrottle
+ self.writedelay = writedelay
+ self.last_read = 0
+ self.last_write = 0
+ self.next_multiplicity = 1.0
+ self.checkdelay = 300 # Check logfile again after this many seconds
+ self.dropdelay = 600 # Ignore processes that have not made
+ # a check in this many seconds
+ self.releasepid = 1200 # Free the process id after this many seconds
+ self.lastwait = 0.0
+ self.delay = 0
+ self.verbosedelay = verbosedelay
+ if multiplydelay:
+ self.checkMultiplicity()
+ self.setDelays()
+
+ def checkMultiplicity(self):
+ """Count running processes for site and set process_multiplicity."""
+ global pid
+ self.lock.acquire()
+ mysite = self.mysite
+ logger.debug("Checking multiplicity: pid = %(pid)s" % globals())
+ try:
+ processes = []
+ my_pid = pid or 1 # start at 1 if global pid not yet set
+ count = 1
+ # open throttle.log
+ try:
+ f = open(self.logfn, 'r')
+ except IOError:
+ if not pid:
+ pass
+ else:
+ raise
+ else:
+ now = time.time()
+ for line in f.readlines():
+ # parse line; format is "pid timestamp site"
+ try:
+ line = line.split(' ')
+ this_pid = int(line[0])
+ ptime = int(line[1].split('.')[0])
+ this_site = line[2].rstrip()
+ except (IndexError, ValueError):
+ continue # Sometimes the file gets corrupted
+ # ignore that line
+ if now - ptime > self.releasepid:
+ continue # process has expired, drop from file
+ if now - ptime <= self.dropdelay \
+ and this_site == mysite \
+ and this_pid != pid:
+ count += 1
+ if this_site != self.mysite or this_pid != pid:
+ processes.append({'pid': this_pid,
+ 'time': ptime,
+ 'site': this_site})
+ if not pid and this_pid >= my_pid:
+ my_pid = this_pid+1 # next unused process id
+
+ if not pid:
+ pid = my_pid
+ self.checktime = time.time()
+ processes.append({'pid': pid,
+ 'time': self.checktime,
+ 'site': mysite})
+ f = open(self.logfn, 'w')
+ processes.sort(key=lambda p:(p['pid'], p['site']))
+ for p in processes:
+ f.write("%(pid)s %(time)s %(site)s\n" % p)
+ f.close()
+ self.process_multiplicity = count
+ if self.verbosedelay:
+ logger.info(
+u"Found %(count)s %(mysite)s processes running, including this one."
+ % locals())
+ finally:
+ self.lock.release()
+
+ def setDelays(self, delay=None, writedelay=None, absolute=False):
+ """Set the nominal delays in seconds. Defaults to config values."""
+ self.lock.acquire()
+ try:
+ maxdelay = self.maxdelay
+ if delay is None:
+ delay = self.mindelay
+ if writedelay is None:
+ writedelay = config.put_throttle
+ if absolute:
+ self.maxdelay = delay
+ self.mindelay = delay
+ self.delay = delay
+ self.writedelay = min(max(self.mindelay, writedelay),
+ self.maxdelay)
+ # Start the delay count now, not at the next check
+ self.last_read = self.last_write = time.time()
+ finally:
+ self.lock.release()
+
+ def getDelay(self, write=False):
+ """Return the actual delay, accounting for multiple processes.
+
+ This value is the maximum wait between reads/writes, not taking
+ account of how much time has elapsed since the last access.
+
+ """
+ global pid
+ if write:
+ thisdelay = self.writedelay
+ else:
+ thisdelay = self.delay
+ if pid: # If set, we're checking for multiple processes
+ if time.time() > self.checktime + self.checkdelay:
+ self.checkMultiplicity()
+ if thisdelay < (self.mindelay * self.next_multiplicity):
+ thisdelay = self.mindelay * self.next_multiplicity
+ elif thisdelay > self.maxdelay:
+ thisdelay = self.maxdelay
+ thisdelay *= self.process_multiplicity
+ return thisdelay
+
+ def waittime(self, write=False):
+ """Return waiting time in seconds if a query would be made right now"""
+ # Take the previous requestsize in account calculating the desired
+ # delay this time
+ thisdelay = self.getDelay(write=write)
+ now = time.time()
+ if write:
+ ago = now - self.last_write
+ else:
+ ago = now - self.last_read
+ if ago < thisdelay:
+ delta = thisdelay - ago
+ return delta
+ else:
+ return 0.0
+
+ def drop(self):
+ """Remove me from the list of running bot processes."""
+ # drop all throttles with this process's pid, regardless of site
+ self.checktime = 0
+ processes = []
+ try:
+ f = open(self.logfn, 'r')
+ except IOError:
+ return
+ else:
+ now = time.time()
+ for line in f.readlines():
+ try:
+ line = line.split(' ')
+ this_pid = int(line[0])
+ ptime = int(line[1].split('.')[0])
+ this_site = line[2].rstrip()
+ except (IndexError,ValueError):
+ continue # Sometimes the file gets corrupted
+ # ignore that line
+ if now - ptime <= self.releasepid \
+ and this_pid != pid:
+ processes.append({'pid': this_pid,
+ 'time': ptime,
+ 'site': this_site})
+ f = open(self.logfn, 'w')
+ processes.sort(key=lambda p:p['pid'])
+ for p in processes:
+ f.write("%(pid)s %(time)s %(site)s\n" % p)
+ f.close()
+
+ def __call__(self, requestsize=1, write=False):
+ """
+ Block the calling program if the throttle time has not expired.
+
+ Parameter requestsize is the number of Pages to be read/written;
+ multiply delay time by an appropriate factor.
+
+ Because this seizes the throttle lock, it will prevent any other
+ thread from writing to the same site until the wait expires.
+
+ """
+ self.lock.acquire()
+ try:
+ wait = self.waittime(write=write)
+ # Calculate the multiplicity of the next delay based on how
+ # big the request is that is being posted now.
+ # We want to add "one delay" for each factor of two in the
+ # size of the request. Getting 64 pages at once allows 6 times
+ # the delay time for the server.
+ self.next_multiplicity = math.log(1+requestsize)/math.log(2.0)
+ # Announce the delay if it exceeds a preset limit
+ if wait > config.noisysleep:
+ logger.info(u"Sleeping for %(wait).1f seconds, %(now)s"
+ % {'wait': wait,
+ 'now': time.strftime("%Y-%m-%d %H:%M:%S",
+ time.localtime())
+ } )
+ time.sleep(wait)
+ if write:
+ self.last_write = time.time()
+ else:
+ self.last_read = time.time()
+ finally:
+ self.lock.release()
+
+ def lag(self, lagtime):
+ """
+ Seize the throttle lock due to server lag.
+
+ This will prevent any thread from accessing this site.
+
+ """
+ started = time.time()
+ self.lock.acquire()
+ try:
+ # start at 1/2 the current server lag time
+ # wait at least 5 seconds but not more than 120 seconds
+ delay = min(max(5, lagtime//2), 120)
+ # account for any time we waited while acquiring the lock
+ wait = delay - (time.time() - started)
+ if wait > 0:
+ if wait > config.noisysleep:
+ logger.info(u"Sleeping for %(wait).1f seconds, %(now)s"
+ % {'wait': wait,
+ 'now': time.strftime("%Y-%m-%d %H:%M:%S",
+ time.localtime())
+ } )
+ time.sleep(wait)
+ finally:
+ self.lock.release()
+
Property changes on: branches/rewrite/pywikibot/throttle.py
___________________________________________________________________
Added: svn:keywords
+ Author Date Id Revision
Added: svn:eol-style
+ native
Modified: branches/rewrite/pywikibot/tools.py
===================================================================
--- branches/rewrite/pywikibot/tools.py 2008-12-16 19:34:48 UTC (rev 6155)
+++ branches/rewrite/pywikibot/tools.py 2008-12-16 19:40:20 UTC (rev 6156)
@@ -1,174 +1,174 @@
-# -*- coding: utf-8 -*-
-"""Miscellaneous helper functions (not wiki-dependent)"""
-#
-# (C) Pywikipedia bot team, 2008
-#
-# Distributed under the terms of the MIT license.
-#
-__version__ = '$Id: $'
-
-import sys
-import threading
-import time
-import Queue
-
-
-class ThreadedGenerator(threading.Thread):
- """Look-ahead generator class.
-
- Runs a generator in a separate thread and queues the results; can
- be called like a regular generator.
-
- Subclasses should override self.generator, I{not} self.run
-
- Important: the generator thread will stop itself if the generator's
- internal queue is exhausted; but, if the calling program does not use
- all the generated values, it must call the generator's stop() method to
- stop the background thread. Example usage:
-
- >>> gen = ThreadedGenerator(target=xrange, args=(20,))
- >>> try:
- ... for data in gen:
- ... print data,
- ... finally:
- ... gen.stop()
- 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
-
- """
-
- def __init__(self, group=None, target=None, name="GeneratorThread",
- args=(), kwargs=None, qsize=65536):
- """Constructor. Takes same keyword arguments as threading.Thread.
-
- target must be a generator function (or other callable that returns
- an iterable object).
-
- @param qsize: The size of the lookahead queue. The larger the qsize,
- the more values will be computed in advance of use (which can eat
- up memory and processor time).
- @type qsize: int
-
- """
- if kwargs is None:
- kwargs = {}
- if target:
- self.generator = target
- if not hasattr(self, "generator"):
- raise RuntimeError("No generator for ThreadedGenerator to run.")
- self.args, self.kwargs = args, kwargs
- threading.Thread.__init__(self, group=group, name=name)
- self.queue = Queue.Queue(qsize)
- self.finished = threading.Event()
-
- def __iter__(self):
- """Iterate results from the queue."""
- if not self.isAlive() and not self.finished.isSet():
- self.start()
- # if there is an item in the queue, yield it, otherwise wait
- while not self.finished.isSet():
- try:
- yield self.queue.get(True, 0.25)
- except Queue.Empty:
- pass
- except KeyboardInterrupt:
- self.stop()
-
- def stop(self):
- """Stop the background thread."""
- self.finished.set()
-
- def run(self):
- """Run the generator and store the results on the queue."""
- self.__gen = self.generator(*self.args, **self.kwargs)
- for result in self.__gen:
- while True:
- if self.finished.isSet():
- return
- try:
- self.queue.put_nowait(result)
- except Queue.Full:
- time.sleep(0.25)
- continue
- break
- # wait for queue to be emptied, then kill the thread
- while not self.finished.isSet() and not self.queue.empty():
- time.sleep(0.25)
- self.stop()
-
-
-def itergroup(iterable, size):
- """Make an iterator that returns lists of (up to) size items from iterable.
-
- Example:
-
- >>> i = itergroup(xrange(25), 10)
- >>> print i.next()
- [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
- >>> print i.next()
- [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
- >>> print i.next()
- [20, 21, 22, 23, 24]
- >>> print i.next()
- Traceback (most recent call last):
- ...
- StopIteration
-
- """
- group = []
- for item in iterable:
- group.append(item)
- if len(group) == size:
- yield group
- group = []
- if group:
- yield group
-
-
-class ThreadList(list):
- """A simple threadpool class to limit the number of simultaneous threads.
-
- Any threading.Thread object can be added to the pool using the append()
- method. If the maximum number of simultaneous threads has not been reached,
- the Thread object will be started immediately; if not, the append() call
- will block until the thread is able to start.
-
- >>> pool = ThreadList(limit=10)
- >>> def work():
- ... time.sleep(1)
- ...
- >>> for x in xrange(20):
- ... pool.append(threading.Thread(target=work))
- ...
-
- """
- def __init__(self, limit=sys.maxint, *args):
- self.limit = limit
- list.__init__(self, *args)
- for item in list(self):
- if not isinstance(threading.Thread, item):
- raise TypeError("Cannot add '%s' to ThreadList" % type(item))
-
- def active_count(self):
- """Return the number of alive threads, and delete all non-alive ones."""
- count = 0
- for item in list(self):
- if item.isAlive():
- count += 1
- else:
- self.remove(item)
- return count
-
- def append(self, thd):
- if not isinstance(thd, threading.Thread):
- raise TypeError("Cannot append '%s' to ThreadList" % type(thd))
- while self.active_count() >= self.limit:
- time.sleep(2)
- list.append(self, thd)
- thd.start()
-
-
-if __name__ == "__main__":
- def _test():
- import doctest
- doctest.testmod()
- _test()
+# -*- coding: utf-8 -*-
+"""Miscellaneous helper functions (not wiki-dependent)"""
+#
+# (C) Pywikipedia bot team, 2008
+#
+# Distributed under the terms of the MIT license.
+#
+__version__ = '$Id$'
+
+import sys
+import threading
+import time
+import Queue
+
+
+class ThreadedGenerator(threading.Thread):
+ """Look-ahead generator class.
+
+ Runs a generator in a separate thread and queues the results; can
+ be called like a regular generator.
+
+ Subclasses should override self.generator, I{not} self.run
+
+ Important: the generator thread will stop itself if the generator's
+ internal queue is exhausted; but, if the calling program does not use
+ all the generated values, it must call the generator's stop() method to
+ stop the background thread. Example usage:
+
+ >>> gen = ThreadedGenerator(target=xrange, args=(20,))
+ >>> try:
+ ... for data in gen:
+ ... print data,
+ ... finally:
+ ... gen.stop()
+ 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19
+
+ """
+
+ def __init__(self, group=None, target=None, name="GeneratorThread",
+ args=(), kwargs=None, qsize=65536):
+ """Constructor. Takes same keyword arguments as threading.Thread.
+
+ target must be a generator function (or other callable that returns
+ an iterable object).
+
+ @param qsize: The size of the lookahead queue. The larger the qsize,
+ the more values will be computed in advance of use (which can eat
+ up memory and processor time).
+ @type qsize: int
+
+ """
+ if kwargs is None:
+ kwargs = {}
+ if target:
+ self.generator = target
+ if not hasattr(self, "generator"):
+ raise RuntimeError("No generator for ThreadedGenerator to run.")
+ self.args, self.kwargs = args, kwargs
+ threading.Thread.__init__(self, group=group, name=name)
+ self.queue = Queue.Queue(qsize)
+ self.finished = threading.Event()
+
+ def __iter__(self):
+ """Iterate results from the queue."""
+ if not self.isAlive() and not self.finished.isSet():
+ self.start()
+ # if there is an item in the queue, yield it, otherwise wait
+ while not self.finished.isSet():
+ try:
+ yield self.queue.get(True, 0.25)
+ except Queue.Empty:
+ pass
+ except KeyboardInterrupt:
+ self.stop()
+
+ def stop(self):
+ """Stop the background thread."""
+ self.finished.set()
+
+ def run(self):
+ """Run the generator and store the results on the queue."""
+ self.__gen = self.generator(*self.args, **self.kwargs)
+ for result in self.__gen:
+ while True:
+ if self.finished.isSet():
+ return
+ try:
+ self.queue.put_nowait(result)
+ except Queue.Full:
+ time.sleep(0.25)
+ continue
+ break
+ # wait for queue to be emptied, then kill the thread
+ while not self.finished.isSet() and not self.queue.empty():
+ time.sleep(0.25)
+ self.stop()
+
+
+def itergroup(iterable, size):
+ """Make an iterator that returns lists of (up to) size items from iterable.
+
+ Example:
+
+ >>> i = itergroup(xrange(25), 10)
+ >>> print i.next()
+ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]
+ >>> print i.next()
+ [10, 11, 12, 13, 14, 15, 16, 17, 18, 19]
+ >>> print i.next()
+ [20, 21, 22, 23, 24]
+ >>> print i.next()
+ Traceback (most recent call last):
+ ...
+ StopIteration
+
+ """
+ group = []
+ for item in iterable:
+ group.append(item)
+ if len(group) == size:
+ yield group
+ group = []
+ if group:
+ yield group
+
+
+class ThreadList(list):
+ """A simple threadpool class to limit the number of simultaneous threads.
+
+ Any threading.Thread object can be added to the pool using the append()
+ method. If the maximum number of simultaneous threads has not been reached,
+ the Thread object will be started immediately; if not, the append() call
+ will block until the thread is able to start.
+
+ >>> pool = ThreadList(limit=10)
+ >>> def work():
+ ... time.sleep(1)
+ ...
+ >>> for x in xrange(20):
+ ... pool.append(threading.Thread(target=work))
+ ...
+
+ """
+ def __init__(self, limit=sys.maxint, *args):
+ self.limit = limit
+ list.__init__(self, *args)
+ for item in list(self):
+ if not isinstance(threading.Thread, item):
+ raise TypeError("Cannot add '%s' to ThreadList" % type(item))
+
+ def active_count(self):
+ """Return the number of alive threads, and delete all non-alive ones."""
+ count = 0
+ for item in list(self):
+ if item.isAlive():
+ count += 1
+ else:
+ self.remove(item)
+ return count
+
+ def append(self, thd):
+ if not isinstance(thd, threading.Thread):
+ raise TypeError("Cannot append '%s' to ThreadList" % type(thd))
+ while self.active_count() >= self.limit:
+ time.sleep(2)
+ list.append(self, thd)
+ thd.start()
+
+
+if __name__ == "__main__":
+ def _test():
+ import doctest
+ doctest.testmod()
+ _test()
Property changes on: branches/rewrite/pywikibot/tools.py
___________________________________________________________________
Added: svn:keywords
+ Author Date Id Revision
Added: svn:eol-style
+ native
Revision: 6155
Author: russblau
Date: 2008-12-16 19:34:48 +0000 (Tue, 16 Dec 2008)
Log Message:
-----------
Configure logging for basic terminal output and logfiles
Modified Paths:
--------------
branches/rewrite/pywikibot/__init__.py
branches/rewrite/pywikibot/bot.py
branches/rewrite/pywikibot/config2.py
Modified: branches/rewrite/pywikibot/__init__.py
===================================================================
--- branches/rewrite/pywikibot/__init__.py 2008-12-16 07:49:38 UTC (rev 6154)
+++ branches/rewrite/pywikibot/__init__.py 2008-12-16 19:34:48 UTC (rev 6155)
@@ -7,7 +7,7 @@
#
# Distributed under the terms of the MIT license.
#
-__version__ = '$Id: $'
+__version__ = '$Id$'
import sys
import logging
@@ -16,7 +16,7 @@
from exceptions import *
import config2 as config
import textlib
-from bot import handleArgs, showHelp
+from bot import *
def deprecate_arg(old_arg, new_arg):
@@ -120,11 +120,6 @@
return raw_input(prompt)
-# Logger configuration
-
-logging.basicConfig()
-logging.getLogger().setLevel(logging.INFO)
-
def set_debug(layer):
"""Set the logger for specified layer to DEBUG level.
@@ -174,7 +169,7 @@
# only need one drop() call because all throttles use the same global pid
try:
_sites[_sites.keys()[0]].throttle.drop()
- logger.info("Dropped throttle(s).")
+ logger.log("VERBOSE", "Dropped throttle(s).")
except IndexError:
pass
Modified: branches/rewrite/pywikibot/bot.py
===================================================================
--- branches/rewrite/pywikibot/bot.py 2008-12-16 07:49:38 UTC (rev 6154)
+++ branches/rewrite/pywikibot/bot.py 2008-12-16 19:34:48 UTC (rev 6155)
@@ -7,14 +7,14 @@
#
# Distributed under the terms of the MIT license.
#
-__version__ = '$Id: $'
+__version__ = '$Id$'
# Note: the intention is to develop this module (at some point) into a Bot
# class definition that can be subclassed to create new, functional bot
# scripts, instead of writing each one from scratch.
-import logging
+import logging, logging.handlers
import os.path
import sys
import pywikibot
@@ -64,7 +64,6 @@
args may be passed as an argument, thereby overriding sys.argv
"""
- global verbose
# get commandline arguments if necessary
if not args:
args = sys.argv[1:]
@@ -72,6 +71,8 @@
# required because the -help option loads the module's docstring and because
# the module name will be used for the filename of the log.
moduleName = calledModuleName()
+ if not moduleName:
+ moduleName = "terminal-interface"
nonGlobalArgs = []
for arg in args:
arg = _decodeArg(arg)
@@ -87,15 +88,23 @@
elif arg.startswith('-pt:'):
config.put_throttle = int(arg[4:])
elif arg == '-log':
- setLogfileStatus(True) #FIXME
+ if moduleName not in config.log:
+ config.log.append(moduleName)
elif arg.startswith('-log:'):
- setLogfileStatus(True, arg[5:]) #FIXME
+ if moduleName not in config.log:
+ config.log.append(moduleName)
+ config.logfilename = arg[5:]
elif arg == '-nolog':
- setLogfileStatus(False) #FIXME
+ if moduleName in config.log:
+ config.log.remove(moduleName)
+ elif arg == "-debug":
+ if moduleName not in config.log:
+ config.log.append(moduleName)
+ config.log.debug_log = True
elif arg == '-verbose' or arg == "-v":
pywikibot.output(u'Pywikipediabot %s' % (version.getversion()))
pywikibot.output(u'Python %s' % (sys.version))
- verbose += 1 # FIXME
+ config.verbose_output += 1
elif arg == '-daemonize':
import daemonize
daemonize.daemonize()
@@ -106,6 +115,35 @@
# the argument is not global. Let the specific bot script care
# about it.
nonGlobalArgs.append(arg)
+
+ # initialize logging system for terminal-based bots
+
+ logging.addLevelName(18, "VERBOSE") # for messages to be displayed on
+ # terminal at "verbose" setting
+ # use INFO for messages to be displayed
+ # even on non-verbose setting
+ logging.addLevelName(24, "STDOUT") # for messages to be displayed to stdout
+ logging.addLevelName(26, "INPUT") # for prompts requiring user response
+
+ logging.basicConfig() # initializes root logger
+ root_logger = logging.getLogger()
+ if config.verbose_output:
+ root_logger.setLevel("VERBOSE")
+ else:
+ root_logger.setLevel(logging.INFO)
+ if moduleName in config.log:
+ if config.logfilename:
+ logfile = config.datafilepath(config.logfilename)
+ else:
+ logfile = config.datafilepath("%s.log" % moduleName)
+ file_handler = logging.handlers.RotatingFileHandler(
+ filename=logfile, maxBytes=2 << 20, backupCount=5)
+ if config.debug_log:
+ file_handler.setLevel(logging.DEBUG)
+ else:
+ file_handler.setLevel("VERBOSE")
+ logging.addHandler(file_handler)
+
return nonGlobalArgs
Modified: branches/rewrite/pywikibot/config2.py
===================================================================
--- branches/rewrite/pywikibot/config2.py 2008-12-16 07:49:38 UTC (rev 6154)
+++ branches/rewrite/pywikibot/config2.py 2008-12-16 19:34:48 UTC (rev 6155)
@@ -216,6 +216,9 @@
# be used to generate so-called warnfiles.
# This setting can be overridden by the -log or -nolog command-line arguments.
log = ['interwiki']
+logfilename = None # defaults to modulename.log
+verbose_output = 0
+debug_log = False
############## INTERWIKI SETTINGS ##############