Revision: 8043
Author: cydeweys
Date: 2010-03-28 19:52:25 +0000 (Sun, 28 Mar 2010)
Log Message:
-----------
Now takes account of whether a target category page is a redirect (and if it is, we abort processing).
Modified Paths:
--------------
trunk/pywikipedia/cfd.py
Modified: trunk/pywikipedia/cfd.py
===================================================================
--- trunk/pywikipedia/cfd.py 2010-03-27 19:29:29 UTC (rev 8042)
+++ trunk/pywikipedia/cfd.py 2010-03-28 19:52:25 UTC (rev 8043)
@@ -16,6 +16,15 @@
import re
import category
+# The locateion of the CFD working page.
+cfdPage = 'Wikipedia:Categories for discussion/Working'
+
+
+# A list of templates that signify that a category is redirected.
+# If the category is redirect, we do NOT want to move articles to it.
+# The safest thing to do here is abort and wait for human intervention.
+redirTemplates = ['Category redirect', 'Categoryredirect', 'CR', 'Catredirect', 'Seecat', 'Cat redirect']
+
# Regular expression declarations
# See the en-wiki CFD working page at [[Wikipedia:Categories for discussion/Working]]
# to see how these work in context. To get this bot working on other wikis you will
@@ -42,7 +51,7 @@
def main():
wikipedia.handleArgs();
- page = wikipedia.Page(wikipedia.getSite(), 'Wikipedia:Categories for discussion/Working')
+ page = wikipedia.Page(wikipedia.getSite(), cfdPage)
# Variable declarations
day = "None"
@@ -88,9 +97,14 @@
summary = "Robot - Speedily moving category " + src + " to " + dest + " per [[WP:CFD|CFD]]."
else:
continue
- robot = category.CategoryMoveRobot(oldCatTitle=src, newCatTitle=dest, batchMode=True,
- editSummary=summary, inPlace=True, moveCatPage=True,
- deleteEmptySourceCat=True)
+ if categoryIsRedirect(dest):
+ summary = 'CANCELED. Destination is redirect: ' + summary
+ wikipedia.output(summary, toStdout=True)
+ robot = None
+ else:
+ robot = category.CategoryMoveRobot(oldCatTitle=src, newCatTitle=dest, batchMode=True,
+ editSummary=summary, inPlace=True, moveCatPage=True,
+ deleteEmptySourceCat=True)
elif (m.check(deletecat, line)):
src = m.result.group(1)
# I currently don't see any reason to handle these two cases separately, though
@@ -112,6 +126,16 @@
summary = ""
robot = None
+# Returns true if the category is a redirected category, meaning we should
+# NOT run the bot and instead let humans handle the exception.
+def categoryIsRedirect(pageTitle):
+ page = wikipedia.Page(wikipedia.getSite(), "Category:" + pageTitle)
+ templates = page.templates(get_redirect=True)
+ for redirTemplate in redirTemplates:
+ if redirTemplate in templates:
+ return True
+ return False
+
# This function grabs the wiki source of a category page and attempts to
# extract a link to the CFD per-day discussion page from the CFD template.
# If the CFD template is not there, it will return the value of the second
Revision: 8040
Author: russblau
Date: 2010-03-26 18:29:19 +0000 (Fri, 26 Mar 2010)
Log Message:
-----------
Refactor and provide better documentation of the user interface functions, hoping this will make it easier for others to contribute in this area.
Modified Paths:
--------------
branches/rewrite/pywikibot/bot.py
branches/rewrite/pywikibot/data/api.py
branches/rewrite/pywikibot/page.py
branches/rewrite/pywikibot/pagegenerators.py
branches/rewrite/pywikibot/site.py
branches/rewrite/pywikibot/userinterfaces/terminal_interface.py
branches/rewrite/scripts/category.py
branches/rewrite/scripts/upload.py
branches/rewrite/tests/site_tests.py
Modified: branches/rewrite/pywikibot/bot.py
===================================================================
--- branches/rewrite/pywikibot/bot.py 2010-03-23 16:28:30 UTC (rev 8039)
+++ branches/rewrite/pywikibot/bot.py 2010-03-26 18:29:19 UTC (rev 8040)
@@ -39,25 +39,9 @@
fromlist=['UI'] )
ui = uiModule.UI()
+
# Logging module configuration
-class MaxLevelFilter(logging.Filter):
- """Filter that only passes records at or below a specific level.
-
- (setting handler level only passes records at or *above* a specified level,
- so this provides the opposite functionality)
-
- """
- def __init__(self, level=None):
- self.level = level
-
- def filter(self, record):
- if self.level:
- return record.levelno <= self.level
- else:
- return True
-
-
class RotatingFileHandler(logging.handlers.RotatingFileHandler):
"""Strip trailing newlines before outputting text to file"""
def format(self, record):
@@ -66,6 +50,13 @@
class LoggingFormatter(logging.Formatter):
+ """Format LogRecords for output to file.
+
+ This formatter *ignores* the 'newline' key of the LogRecord, because
+ every record written to a file must end with a newline, regardless of
+ whether the output to the user's console does.
+
+ """
def formatException(self, ei):
"""
Make sure that the exception trace is converted to unicode:
@@ -85,15 +76,118 @@
return strExc + '\n'
+# Initialize the handlers and formatters for the logging system.
+#
+# This relies on the global variable 'ui' which is a UserInterface object
+# defined in the 'userinterface' subpackage.
+#
+# The UserInterface object must define its own init_handlers() method
+# which takes the root logger as its only argument, and which adds to that
+# logger whatever handlers and formatters are needed to process output and
+# display it to the user. The default (terminal) interface sends level
+# STDOUT to sys.stdout (as all interfaces should) and sends all other
+# levels to sys.stderr; levels WARNING and above are labeled with the
+# level name.
+#
+# UserInterface objects must also define methods input(), inputChoice(),
+# editText(), and askForCaptcha(), all of which are documented in
+# userinterfaces/terminal_interface.py
+
+_handlers_initialized = False
+
+def init_handlers(strm=None):
+ """Initialize logging system for terminal-based bots.
+
+ This function must be called before using pywikibot.output(); and must
+ be called again if the destination stream is changed.
+
+ @param strm: Output stream. If None, re-uses the last stream if one
+ was defined, otherwise uses sys.stderr
+
+ """
+ # Note: this function is called by handleArgs(), so it should normally
+ # not need to be called explicitly
+
+ # All user output is routed through the logging module.
+ # Each type of output is handled by an appropriate handler object.
+ # This structure is used to permit eventual development of other
+ # user interfaces (GUIs) without modifying the core bot code.
+ # The following output levels are defined:
+ # DEBUG - only for file logging; debugging messages
+ # STDOUT - output that must be sent to sys.stdout (for bots that may
+ # have their output redirected to a file or other destination)
+ # VERBOSE - optional progress information for display to user
+ # INFO - normal (non-optional) progress information for display to user
+ # INPUT - prompts requiring user response
+ # WARN - user warning messages
+ # ERROR - user error messages
+ # CRITICAL - fatal error messages
+ # Accordingly, do ''not'' use print statements in bot code; instead,
+ # use pywikibot.output function.
+
+ global _handlers_initialized
+
+ moduleName = calledModuleName()
+ if not moduleName:
+ moduleName = "terminal-interface"
+
+ logging.addLevelName(VERBOSE, "VERBOSE")
+ # for messages to be displayed on terminal at "verbose" setting
+ # use INFO for messages to be displayed even on non-verbose setting
+ logging.addLevelName(STDOUT, "STDOUT")
+ # for messages to be displayed to stdout
+ logging.addLevelName(INPUT, "INPUT")
+ # for prompts requiring user response
+
+ root_logger = logging.getLogger("pywiki")
+ root_logger.setLevel(DEBUG+1) # all records except DEBUG go to logger
+ root_logger.handlers = [] # remove any old handlers
+
+ # configure handler(s) for display to user interface
+ ui.init_handlers(root_logger)
+
+ # if user has enabled file logging, configure file handler
+ if moduleName in config.log or '*' in config.log:
+ if config.logfilename:
+ logfile = config.datafilepath(config.logfilename)
+ else:
+ logfile = config.datafilepath("%s-bot.log" % moduleName)
+ file_handler = RotatingFileHandler(
+ filename=logfile, maxBytes=2 << 20, backupCount=5)
+
+ file_handler.setLevel(DEBUG)
+ form = LoggingFormatter(
+ fmt="%(asctime)s %(caller_file)18s, %(caller_line)4s "
+ "in %(caller_name)18s: %(levelname)-8s %(message)s",
+ datefmt="%Y-%m-%d %H:%M:%S"
+ )
+ file_handler.setFormatter(form)
+ root_logger.addHandler(file_handler)
+ # Turn on debugging for each component requested by user
+ # or for all components if nothing was specified
+ for component in config.debug_log:
+ if component:
+ debuglogger = logging.getLogger("pywiki."+component)
+ else:
+ debuglogger = logging.getLogger("pywiki")
+ debuglogger.setLevel(DEBUG)
+ debuglogger.addHandler(file_handler)
+
+ _handlers_initialized = True
+
+
# User output/logging functions
-# Five output functions are defined. Each requires a unicode or string
+# Six output functions are defined. Each requires a unicode or string
# argument. All of these functions generate a message to the log file if
# logging is enabled ("-log" or "-debug" command line arguments).
-# The functions output(), warning(), and error() all display a message to the
-# user through the logger object; the only difference is the priority level,
-# which can be used by the application layer to alter the display.
+# The functions output(), stdout(), warning(), and error() all display a
+# message to the user through the logger object; the only difference is the
+# priority level, which can be used by the application layer to alter the
+# display. The stdout() function should be used only for data that is
+# the "result" of a script, as opposed to information messages to the
+# user.
# The function log() by default does not display a message to the user, but
# this can be altered by using the "-verbose" command line option.
@@ -108,39 +202,40 @@
try:
raise Exception
except:
- # go back two levels, one for _fmtoutput and one for whatever called it
+ # go back two levels, one for logoutput and one for whatever called it
return sys.exc_traceback.tb_frame.f_back.f_back
if hasattr(sys, '_getframe'):
# less portable but more efficient
currentframe = lambda: sys._getframe(3)
- # frame0 is this lambda, frame1 is _fmtoutput() in this module,
+ # frame0 is this lambda, frame1 is logoutput() in this module,
# frame2 is the convenience function (output(), etc.)
# so frame3 is whatever called the convenience function
# done filching
-def _fmtoutput(text, decoder=None, newline=True, toStdout=False,
- _level=INFO, _logger=""):
+def logoutput(text, decoder=None, newline=True, _level=INFO, _logger="",
+ **kwargs):
"""Format output and send to the logging module.
Backend function used by all the user-output convenience functions.
"""
if _logger:
- path = "pywiki." + _logger
+ logger = logging.getLogger("pywiki." + _logger)
else:
- path = "pywiki"
+ logger = logging.getLogger("pywiki")
# make sure logging system has been initialized
if not _handlers_initialized:
- init_handlers(strm=ui.output_stream)
+ init_handlers()
frame = currentframe()
module = os.path.basename(frame.f_code.co_filename)
context = {'caller_name': frame.f_code.co_name,
'caller_file': module,
- 'caller_line': frame.f_lineno}
+ 'caller_line': frame.f_lineno,
+ 'newline': ("\n" if newline else "")}
if decoder:
text = unicode(text, decoder)
@@ -155,18 +250,15 @@
text = unicode(text, 'utf-8')
except UnicodeDecodeError:
text = unicode(text, 'iso8859-1')
- if newline:
- text += "\n"
- if toStdout:
- _level = STDOUT
- logger = logging.getLogger(path)
- ui.output(text, logger, _level, context)
-def output(text, decoder=None, newline=True, toStdout=False):
+ logger.log(_level, text, extra=context, **kwargs)
+
+def output(text, decoder=None, newline=True, toStdout=False, **kwargs):
"""Output a message to the user via the userinterface.
Works like print, but uses the encoding used by the user's console
(console_encoding in the configuration file) instead of ASCII.
+
If decoder is None, text should be a unicode string. Otherwise it
should be encoded in the given encoding.
@@ -180,22 +272,37 @@
consist of the escape character \03 and the color name in curly braces,
e. g. \03{lightpurple}. \03{default} resets the color.
+ Other keyword arguments are passed unchanged to the logger; so far, the
+ only argument that is useful is "exc_info=True", which causes the
+ log message to include an exception traceback.
+
"""
- _fmtoutput(text, decoder, newline, toStdout, INFO)
+ if toStdout: # maintained for backwards-compatibity only
+ logoutput(text, decoder, newline, STDOUT, **kwargs)
+ else:
+ logoutput(text, decoder, newline, INFO, **kwargs)
-def warning(text, decoder=None, newline=True, toStdout=False):
- _fmtoutput(text, decoder, newline, toStdout, WARNING)
+def stdout(text, decoder=None, newline=True, **kwargs):
+ """Output script results to the user via the userinterface."""
+ logoutput(text, decoder, newline, STDOUT, **kwargs)
-def error(text, decoder=None, newline=True, toStdout=False):
- _fmtoutput(text, decoder, newline, toStdout, ERROR)
+def warning(text, decoder=None, newline=True, **kwargs):
+ """Output a warning message to the user via the userinterface."""
+ logoutput(text, decoder, newline, WARNING, **kwargs)
-def log(text, decoder=None, newline=True, toStdout=False):
- _fmtoutput(text, decoder, newline, toStdout, VERBOSE)
+def error(text, decoder=None, newline=True, **kwargs):
+ """Output an error message to the user via the userinterface."""
+ logoutput(text, decoder, newline, ERROR, **kwargs)
-def debug(text, layer, decoder=None, newline=True, toStdout=False):
- _fmtoutput(text, decoder, newline, toStdout, DEBUG, layer)
+def log(text, decoder=None, newline=True, **kwargs):
+ """Output a record to the log file."""
+ logoutput(text, decoder, newline, VERBOSE, **kwargs)
+def debug(text, layer, decoder=None, newline=True, **kwargs):
+ """Output a debug record to the log file."""
+ logoutput(text, decoder, newline, DEBUG, layer, **kwargs)
+
# User input functions
def input(question, password=False):
@@ -212,7 +319,7 @@
"""
# make sure logging system has been initialized
if not _handlers_initialized:
- init_handlers(strm=ui.output_stream)
+ init_handlers()
data = ui.input(question, password)
return data
@@ -237,123 +344,12 @@
"""
# make sure logging system has been initialized
if not _handlers_initialized:
- init_handlers(strm=ui.output_stream)
+ init_handlers()
data = ui.inputChoice(question, answers, hotkeys, default).lower()
return data
-_handlers_initialized = False
-
-def init_handlers(strm=None):
- """Initialize logging system for terminal-based bots.
-
- This function must be called before using pywikibot.output(); and must
- be called again if the destination stream is changed.
-
- @param strm: Output stream. If None, re-uses the last stream if one
- was defined, otherwise uses sys.stderr
-
- """
- # Note: this function is called by handleArgs(), so it should normally
- # not need to be called explicitly
-
- # All user output is routed through the logging module.
- # Each type of output is handled by an appropriate handler object.
- # This structure is used to permit eventual development of other
- # user interfaces (GUIs) without modifying the core bot code.
- # The following output levels are defined:
- # DEBUG - only for file logging; debugging messages
- # STDOUT - output that must be sent to sys.stdout (for bots that may
- # have their output redirected to a file or other destination)
- # VERBOSE - optional progress information for display to user
- # INFO - normal (non-optional) progress information for display to user
- # INPUT - prompts requiring user response
- # WARN - user warning messages
- # ERROR - user error messages
- # CRITICAL - fatal error messages
- # Accordingly, do ''not'' use print statements in bot code; instead,
- # use pywikibot.output function.
-
- global _handlers_initialized
-
- global _stream
- if strm:
- _stream = strm
- else:
- try:
- _stream
- except NameError:
- _stream = sys.stderr
- moduleName = calledModuleName()
- if not moduleName:
- moduleName = "terminal-interface"
-
- logging.addLevelName(VERBOSE, "VERBOSE")
- # for messages to be displayed on terminal at "verbose" setting
- # use INFO for messages to be displayed even on non-verbose setting
- logging.addLevelName(STDOUT, "STDOUT")
- # for messages to be displayed to stdout
- logging.addLevelName(INPUT, "INPUT")
- # for prompts requiring user response
-
- root_logger = logging.getLogger("pywiki")
- root_logger.setLevel(DEBUG+1) # all records except DEBUG go to logger
- root_logger.handlers = [] # remove any old handlers
-
- # configure default handler for display to user interface
- default_handler = ui.OutputHandlerClass(strm=_stream)
- if config.verbose_output:
- default_handler.setLevel(VERBOSE)
- else:
- default_handler.setLevel(INFO)
- default_handler.addFilter(MaxLevelFilter(INPUT))
- default_handler.setFormatter(LoggingFormatter(fmt="%(message)s"))
- root_logger.addHandler(default_handler)
-
- # if user has enabled file logging, configure file handler
- if moduleName in config.log or '*' in config.log:
- if config.logfilename:
- logfile = config.datafilepath(config.logfilename)
- else:
- logfile = config.datafilepath("%s-bot.log" % moduleName)
- file_handler = RotatingFileHandler(
- filename=logfile, maxBytes=2 << 20, backupCount=5)
-
- file_handler.setLevel(DEBUG)
- form = LoggingFormatter(
- fmt="%(asctime)s %(caller_file)18s, %(caller_line)4s "
- "in %(caller_name)18s: %(levelname)-8s %(message)s",
- datefmt="%Y-%m-%d %H:%M:%S"
- )
- file_handler.setFormatter(form)
- root_logger.addHandler(file_handler)
- # Turn on debugging for each component requested by user
- # or for all components if nothing was specified
- for component in config.debug_log:
- if component:
- debuglogger = logging.getLogger("pywiki."+component)
- else:
- debuglogger = logging.getLogger("pywiki")
- debuglogger.setLevel(DEBUG)
- debuglogger.addHandler(file_handler)
-
- # handler for level STDOUT
- output_handler = ui.OutputHandlerClass(strm=sys.stdout)
- output_handler.setLevel(STDOUT)
- output_handler.addFilter(MaxLevelFilter(STDOUT))
- output_handler.setFormatter(LoggingFormatter(fmt="%(message)s"))
- root_logger.addHandler(output_handler)
-
- # handler for levels WARNING and higher
- warning_handler = ui.OutputHandlerClass(strm=_stream)
- warning_handler.setLevel(logging.WARNING)
- warning_handler.setFormatter(
- LoggingFormatter(fmt="%(levelname)s: %(message)s"))
- root_logger.addHandler(warning_handler)
-
- _handlers_initialized = True
-
# Command line parsing and help
def calledModuleName():
@@ -487,7 +483,7 @@
if username:
config.usernames[config.family][config.mylang] = username
- init_handlers(strm=ui.output_stream)
+ init_handlers()
if config.verbose_output:
import re
@@ -542,7 +538,7 @@
-nolog Disable the logfile (if it is enabled by default).
-debug:item Enable the logfile and include extensive debugging data
--debug for component "item" (or all components if the second form
+-debug for component "item" (for all components if the second form
is used).
-putthrottle:n Set the minimum time (in seconds) the bot will wait between
@@ -558,14 +554,14 @@
if hasattr(module, 'docuReplacements'):
for key, value in module.docuReplacements.iteritems():
helpText = helpText.replace(key, value.strip('\n\r'))
- pywikibot.output(helpText, toStdout=True) # output to STDOUT
+ pywikibot.stdout(helpText) # output to STDOUT
except Exception:
if modname:
- pywikibot.output(u'Sorry, no help available for %s' % modname,
- toStdout=True)
- logger.exception('showHelp:')
- pywikibot.output(globalHelp, toStdout=True)
+ pywikibot.stdout(u'Sorry, no help available for %s' % modname)
+# pywikibot.log('showHelp: %%(exception)s')
+ pywikibot.stdout(globalHelp)
+
class Bot(object):
"""
Generic Bot to be subclassed
Modified: branches/rewrite/pywikibot/data/api.py
===================================================================
--- branches/rewrite/pywikibot/data/api.py 2010-03-23 16:28:30 UTC (rev 8039)
+++ branches/rewrite/pywikibot/data/api.py 2010-03-26 18:29:19 UTC (rev 8040)
@@ -33,9 +33,6 @@
_modules = {} # cache for retrieved API parameter information
-##logger = logging.getLogger("pywiki.data.api")
-##print "level =", logger.getEffectiveLevel()
-##
class APIError(pywikibot.Error):
"""The wiki site returned an error message."""
def __init__(self, code, info, **kwargs):
@@ -195,7 +192,9 @@
self.params[key] = self.params[key].encode(
self.site.encoding())
except Exception:
- logger.exception("key=%s, params=%s\n" % (key, self.params[key]))
+ pywikibot.error(
+u"http_params: Key '%s' could not be encoded to '%s'; params=%r"
+ % (key, self.site.encoding(), self.params[key]))
return urllib.urlencode(self.params)
def __str__(self):
Modified: branches/rewrite/pywikibot/page.py
===================================================================
--- branches/rewrite/pywikibot/page.py 2010-03-23 16:28:30 UTC (rev 8039)
+++ branches/rewrite/pywikibot/page.py 2010-03-26 18:29:19 UTC (rev 8040)
@@ -745,8 +745,7 @@
raise
# TODO: other "expected" error types to catch?
except pywikibot.Error, err:
- logger.exception(u"Error saving page %s\n" % link)
- pywikibot.output(u"")
+ pywikibot.log(u"Error saving page %s\n" % link, exc_info=True)
if not callback:
raise pywikibot.PageNotSaved(link)
if callback:
@@ -1378,12 +1377,12 @@
def removeImage(self, image, put=False, summary=None, safe=True):
"""Old method to remove all instances of an image from page."""
- logger.warning(u"Page.removeImage() is no longer supported.")
+ pywikibot.warning(u"Page.removeImage() is no longer supported.")
def replaceImage(self, image, replacement=None, put=False, summary=None,
safe=True):
"""Old method to replace all instances of an image with another."""
- logger.warning(u"Page.replaceImage() is no longer supported.")
+ pywikibot.warning(u"Page.replaceImage() is no longer supported.")
class ImagePage(Page):
Modified: branches/rewrite/pywikibot/pagegenerators.py
===================================================================
--- branches/rewrite/pywikibot/pagegenerators.py 2010-03-23 16:28:30 UTC (rev 8039)
+++ branches/rewrite/pywikibot/pagegenerators.py 2010-03-26 18:29:19 UTC (rev 8040)
@@ -1071,11 +1071,11 @@
gen = genFactory.getCombinedGenerator()
if gen:
for page in gen:
- pywikibot.output(page.title(), toStdout=True)
+ pywikibot.stdout(page.title())
else:
pywikibot.showHelp()
except Exception:
- pywikibot.logging.exception("")
+ pywikibot.error("Fatal error", exc_info=True)
finally:
pywikibot.stopme()
Modified: branches/rewrite/pywikibot/site.py
===================================================================
--- branches/rewrite/pywikibot/site.py 2010-03-23 16:28:30 UTC (rev 8039)
+++ branches/rewrite/pywikibot/site.py 2010-03-26 18:29:19 UTC (rev 8040)
@@ -61,10 +61,10 @@
myfamily = __import__("%s_family" % fam)
except ImportError:
if fatal:
- logger.exception(u"""\
+ pywikibot.error(u"""\
Error importing the %s family. This probably means the family
does not exist. Also check your configuration file."""
- % fam)
+ % fam, exc_info=True)
sys.exit(1)
else:
raise Error("Family %s does not exist" % fam)
Modified: branches/rewrite/pywikibot/userinterfaces/terminal_interface.py
===================================================================
--- branches/rewrite/pywikibot/userinterfaces/terminal_interface.py 2010-03-23 16:28:30 UTC (rev 8039)
+++ branches/rewrite/pywikibot/userinterfaces/terminal_interface.py 2010-03-26 18:29:19 UTC (rev 8040)
@@ -12,6 +12,7 @@
import threading
import pywikibot
from pywikibot import config
+from pywikibot.bot import DEBUG, VERBOSE, INFO, STDOUT, INPUT, WARNING
from pywikibot.userinterfaces import transliteration
@@ -114,22 +115,47 @@
colorTagR = re.compile('\03{(?P<name>%s)}' % '|'.join(windowsColors.keys()))
-class UI:
- def __init__(self):
- self.writelock = threading.RLock()
- self.OutputHandlerClass = TerminalHandler
- self.output_stream = sys.stderr
+class UI(object):
+ def init_handlers(self, root_logger):
+ """Initialize the handlers for user output.
- def output(self, text, logger, level=logging.INFO, context=None):
- """Send text to the logger for output to terminal."""
- self.writelock.acquire()
- try:
- logger.log(level, text, extra=context)
- finally:
- self.writelock.release()
+ This method initializes handler(s) for output levels VERBOSE (if
+ enabled by config.verbose_output), INFO, STDOUT, WARNING, ERROR,
+ and CRITICAL. STDOUT writes its output to sys.stdout; all the
+ others write theirs to sys.stderr.
+ """
+ # default handler for display to terminal
+ default_handler = TerminalHandler(strm=sys.stderr)
+ if config.verbose_output:
+ default_handler.setLevel(VERBOSE)
+ else:
+ default_handler.setLevel(INFO)
+ # this handler ignores levels above INPUT
+ default_handler.addFilter(MaxLevelFilter(INPUT))
+ default_handler.setFormatter(
+ TerminalFormatter(fmt="%(message)s%(newline)s"))
+ root_logger.addHandler(default_handler)
+
+ # handler for level STDOUT
+ output_handler = TerminalHandler(strm=sys.stdout)
+ output_handler.setLevel(STDOUT)
+ output_handler.addFilter(MaxLevelFilter(STDOUT))
+ output_handler.setFormatter(
+ TerminalFormatter(fmt="%(message)s%(newline)s"))
+ root_logger.addHandler(output_handler)
+
+ # handler for levels WARNING and higher
+ warning_handler = TerminalHandler(strm=sys.stderr)
+ warning_handler.setLevel(logging.WARNING)
+ warning_handler.setFormatter(
+ TerminalFormatter(fmt="%(levelname)s: %(message)s%(newline)s"))
+ root_logger.addHandler(warning_handler)
+
def input(self, question, password = False):
"""
+ Ask the user a question and return the answer.
+
Works like raw_input(), but returns a unicode string instead of ASCII.
Unlike raw_input, this function automatically adds a space after the
@@ -142,23 +168,25 @@
# While we're waiting for user input,
# we don't want terminal writes from other Threads
- self.writelock.acquire()
- pywikibot.bot._fmtoutput(question + ' ', newline=False,
- _level=pywikibot.INPUT)
-
+ TerminalHandler.sharedlock.acquire()
try:
+ pywikibot.logoutput(question + ' ', newline=False,
+ _level=pywikibot.INPUT)
if password:
import getpass
text = getpass.getpass('')
else:
text = raw_input()
finally:
- self.writelock.release()
+ TerminalHandler.sharedlock.release()
text = unicode(text, config.console_encoding)
return text
def inputChoice(self, question, options, hotkeys, default=None):
+ """
+ Ask the user a question with a predefined list of acceptable answers.
+ """
options = options[:] # we don't want to edit the passed parameter
for i in range(len(options)):
option = options[i]
@@ -197,13 +225,15 @@
return answer
def editText(self, text, jumpIndex = None, highlight = None):
- """
+ """Return the text as edited by the user.
+
Uses a Tkinter edit box because we don't have a console editor
Parameters:
* text - a Unicode string
* jumpIndex - an integer: position at which to put the caret
* highlight - a substring; each occurence will be highlighted
+
"""
try:
import gui
@@ -214,6 +244,7 @@
return editor.edit(text, jumpIndex=jumpIndex, highlight=highlight)
def askForCaptcha(self, url):
+ """Show the user a CAPTCHA image and return the answer."""
try:
import webbrowser
pywikibot.output(u'Opening CAPTCHA in your web browser...')
@@ -237,6 +268,10 @@
"""
+ # create a class-level lock that can be shared by all instances
+ import threading
+ sharedlock = threading.RLock()
+
def __init__(self, strm=None):
"""Initialize the handler.
@@ -244,6 +279,10 @@
"""
logging.Handler.__init__(self)
+ # replace Handler's instance-specific lock with the shared class lock
+ # to ensure that only one instance of this handler can write to
+ # the console at a time
+ self.lock = TerminalHandler.sharedlock
if strm is None:
strm = sys.stderr
self.stream = strm
@@ -383,3 +422,24 @@
self.emitColorizedInUnix(record, text)
else:
self.emit_raw(record, text)
+
+
+class TerminalFormatter(logging.Formatter):
+ pass
+
+
+class MaxLevelFilter(logging.Filter):
+ """Filter that only passes records at or below a specific level.
+
+ (setting handler level only passes records at or *above* a specified level,
+ so this provides the opposite functionality)
+
+ """
+ def __init__(self, level=None):
+ self.level = level
+
+ def filter(self, record):
+ if self.level:
+ return record.levelno <= self.level
+ else:
+ return True
Modified: branches/rewrite/scripts/category.py
===================================================================
--- branches/rewrite/scripts/category.py 2010-03-23 16:28:30 UTC (rev 8039)
+++ branches/rewrite/scripts/category.py 2010-03-26 18:29:19 UTC (rev 8040)
@@ -1048,7 +1048,7 @@
try:
main()
except pywikibot.Error:
- pywikibot.logging.exception("Fatal error:")
+ pywikibot.error("Fatal error:", exc_info=True)
finally:
catDB.dump()
pywikibot.stopme()
Modified: branches/rewrite/scripts/upload.py
===================================================================
--- branches/rewrite/scripts/upload.py 2010-03-23 16:28:30 UTC (rev 8039)
+++ branches/rewrite/scripts/upload.py 2010-03-26 18:29:19 UTC (rev 8040)
@@ -221,7 +221,7 @@
return
except Exception, e:
- pywikibot.logger.exception("Upload error: " + str(e))
+ pywikibot.error("Upload error: ", exc_info=True)
else:
#No warning, upload complete.
Modified: branches/rewrite/tests/site_tests.py
===================================================================
--- branches/rewrite/tests/site_tests.py 2010-03-23 16:28:30 UTC (rev 8039)
+++ branches/rewrite/tests/site_tests.py 2010-03-26 18:29:19 UTC (rev 8040)
@@ -14,8 +14,6 @@
import pywikibot
import warnings
-logger = pywikibot.logging.getLogger("wiki.site.tests")
-
mysite = pywikibot.Site()
mainpage = pywikibot.Page(pywikibot.Link("Main Page", mysite))
imagepage = iter(mainpage.imagelinks()).next() # 1st image on main page
@@ -57,7 +55,7 @@
def testLanguageMethods(self):
"""Test cases for languages() and related methods"""
-
+
langs = mysite.languages()
self.assertType(langs, list)
self.assertTrue(mysite.code in langs)
@@ -112,7 +110,7 @@
def testApiMethods(self):
"""Test generic ApiSite methods"""
-
+
self.assertType(mysite.logged_in(), bool)
self.assertType(mysite.logged_in(True), bool)
self.assertType(mysite.userinfo, dict)
@@ -129,7 +127,7 @@
self.assertType(mysite.has_group("bots", True), bool)
self.assertFalse(mysite.has_group("nonexistent_group", True))
except pywikibot.NoUsername:
- logger.warn(
+ pywikibot.warning(
"Cannot test Site methods for sysop; no sysop account configured.")
for msg in ("1movedto2", "about", "aboutpage", "aboutsite",
"accesskey-n-portal"):
@@ -147,7 +145,7 @@
def testPageMethods(self):
"""Test ApiSite methods for getting page-specific info"""
-
+
self.assertType(mysite.page_exists(mainpage), bool)
self.assertType(mysite.page_restrictions(mainpage), dict)
self.assertType(mysite.page_can_be_edited(mainpage), bool)
@@ -164,7 +162,7 @@
def testTokens(self):
"""Test ability to get page tokens"""
-
+
for ttype in ("edit", "move"): # token types for non-sysops
self.assertType(mysite.token(mainpage, ttype), basestring)
self.assertRaises(KeyError, mysite.token, mainpage, "invalidtype")
@@ -184,7 +182,7 @@
def testLinkMethods(self):
"""Test site methods for getting links to and from a page"""
-
+
backlinks = set(mysite.pagebacklinks(mainpage, namespaces=[0]))
# only non-redirects:
filtered = set(mysite.pagebacklinks(mainpage, namespaces=0,
@@ -258,7 +256,7 @@
def testLoadRevisions(self):
"""Test the site.loadrevisions() method"""
-
+
mysite.loadrevisions(mainpage)
self.assertTrue(hasattr(mainpage, "_revid"))
self.assertTrue(hasattr(mainpage, "_revisions"))
@@ -334,7 +332,7 @@
def testAllLinks(self):
"""Test the site.alllinks() method"""
-
+
fwd = list(mysite.alllinks(total=10))
self.assertTrue(len(fwd) <= 10)
self.assertTrue(all(isinstance(link, pywikibot.Page) for link in fwd))
@@ -361,7 +359,7 @@
def testAllCategories(self):
"""Test the site.allcategories() method"""
-
+
ac = list(mysite.allcategories(total=10))
self.assertTrue(len(ac) <= 10)
self.assertTrue(all(isinstance(cat, pywikibot.Category)
@@ -379,7 +377,7 @@
def testAllUsers(self):
"""Test the site.allusers() method"""
-
+
au = list(mysite.allusers(total=10))
self.assertTrue(len(au) <= 10)
for user in au:
@@ -828,7 +826,7 @@
try:
mysite.login(True)
except pywikibot.NoUsername:
- logger.warn(
+ pywikibot.warning(
"Cannot test Site.deleted_revs; no sysop account configured.")
return
dr = list(mysite.deletedrevs(total=10, page=mainpage))
@@ -909,7 +907,6 @@
if __name__ == '__main__':
-# pywikibot.logging.getLogger("").setLevel(pywikibot.logging.DEBUG)
try:
try:
unittest.main()
Revision: 8039
Author: xqt
Date: 2010-03-23 16:28:30 +0000 (Tue, 23 Mar 2010)
Log Message:
-----------
new array formatting
Modified Paths:
--------------
trunk/pywikipedia/maintenance/wikimedia_sites.py
Modified: trunk/pywikipedia/maintenance/wikimedia_sites.py
===================================================================
--- trunk/pywikipedia/maintenance/wikimedia_sites.py 2010-03-23 07:42:00 UTC (rev 8038)
+++ trunk/pywikipedia/maintenance/wikimedia_sites.py 2010-03-23 16:28:30 UTC (rev 8039)
@@ -54,19 +54,15 @@
else:
wikipedia.output(u"The lists don't match, the new list is:")
text = u' self.languages_by_size = [\r\n'
- line = ' '
- index = 0
+ line = ' '
for lang in new:
- index += 1
- if index > 1:
- line += u' '
- line += u"'%s'," % lang
- if index == 10:
+ if len(line)+len(lang) <= 76:
+ line += u" '%s'," % lang
+ else:
text += u'%s\r\n' % line
- line = ' '
- index = 0
- if index > 0:
- text += u'%s\r\n' % line
+ line = ' '
+ line += u" '%s'," % lang
+ text += u'%s\r\n' % line
text += u' ]'
wikipedia.output(text)
family_file_name = '../families/%s_family.py' % family
Revision: 8036
Author: xqt
Date: 2010-03-22 16:53:06 +0000 (Mon, 22 Mar 2010)
Log Message:
-----------
deactivate getall() via api. Some pages are still missing.
Modified Paths:
--------------
trunk/pywikipedia/wikipedia.py
Modified: trunk/pywikipedia/wikipedia.py
===================================================================
--- trunk/pywikipedia/wikipedia.py 2010-03-22 14:49:07 UTC (rev 8035)
+++ trunk/pywikipedia/wikipedia.py 2010-03-22 16:53:06 UTC (rev 8036)
@@ -680,6 +680,8 @@
data = query.GetData(params, self.site(), sysop=sysop)
if 'error' in data:
raise RuntimeError("API query error: %s" % data)
+ if not 'pages' in data['query']:
+ raise RuntimeError("API query error, no pages found: %s" % data)
pageInfo = data['query']['pages'].values()[0]
if data['query']['pages'].keys()[0] == "-1":
if 'missing' in pageInfo:
@@ -3664,7 +3666,7 @@
def run(self):
if self.pages:
# Sometimes query does not contains revisions
- if self.site.has_api():
+ if self.site.has_api() and debug:
while True:
try:
data = self.getDataApi()
@@ -4027,7 +4029,7 @@
# TODO: why isn't this a Site method?
pages = list(pages) # if pages is an iterator, we need to make it a list
output(u'Getting %d pages %s from %s...'
- % (len(pages), iif(site.has_api(), u'via API', u''), site))
+ % (len(pages), iif(site.has_api() and debug, u'via API', u''), site))
limit = config.special_page_limit / 4 # default is 500/4, but It might have good point for server.
if len(pages) > limit:
# separate export pages for bulk-retrieve
@@ -6235,9 +6237,9 @@
params = {
'action' : 'query',
'list' : 'allpages',
- 'aplimit' : config.special_page_limit,
+ 'aplimit' : config.special_page_limit,
'apnamespace': namespace,
- 'apfrom': start
+ 'apfrom' : start
}
if not includeredirects:
@@ -6250,7 +6252,9 @@
if throttle:
get_throttle()
data = query.GetData(params, self)
-
+ if 'warnings' in data:
+ warning = data['warnings']['allpages']['*']
+ raise str(warning)
count = 0
for p in data['query']['allpages']:
count += 1
@@ -7184,6 +7188,7 @@
# global debug option for development purposes. Normally does nothing.
elif arg == '-debug':
debug = True
+ config.special_page_limit = 500
else:
# the argument is not global. Let the specific bot script care
# about it.