http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11406
Revision: 11406
Author: xqt
Date: 2013-04-21 12:03:57 +0000 (Sun, 21 Apr 2013)
Log Message:
-----------
- Replace marked nested templates in reverse order, fix for bug #3611198;
- speed up replacement for duplicate templates
- bugfix for pywikipedia.Error
Modified Paths:
--------------
trunk/pywikipedia/pywikibot/textlib.py
Modified: trunk/pywikipedia/pywikibot/textlib.py
===================================================================
--- trunk/pywikipedia/pywikibot/textlib.py 2013-04-21 10:55:57 UTC (rev 11405)
+++ trunk/pywikipedia/pywikibot/textlib.py 2013-04-21 12:03:57 UTC (rev 11406)
@@ -140,16 +140,23 @@
text = text.replace(item, '%s%d%s' % (marker2, count, marker2))
values[count] = item
inside = {}
- count = 0
+ seen = set()
while TEMP_REGEX.search(text) is not None:
for m in TEMP_REGEX.finditer(text):
- count += 1
item = m.group()
+ if item in seen:
+ continue # speed up
+ seen.add(item)
+ count = len(seen)
text = text.replace(item, '%s%d%s' % (marker1, count, marker1))
# Make sure stored templates don't contain markers
- for m2 in Rmarker1.finditer(item):
- item = item.replace(m2.group(), inside[int(m2.group(1))])
+ # We replace the last item first, otherwise inside templates
+ # like {{A{{B}}{{C}}1{{D}}}} could fail
+ for i in range(count - 1, 0, -1):
+ item = item.replace(u'%(mark)s%(number)s%(mark)s'
+ % {'mark': marker1, 'number': i},
+ inside[i])
for m2 in Rmarker2.finditer(item):
item = item.replace(m2.group(), values[int(m2.group(1))])
inside[count] = item
@@ -745,7 +752,7 @@
if site is None:
site = pywikibot.getSite()
if site.sitename() == 'wikipedia:de' and "{{Personendaten" in oldtext:
- raise Error("""\
+ raise pywikibot.Error("""\
The PyWikipediaBot is no longer allowed to touch categories on the German
Wikipedia on pages that contain the Personendaten template because of the
non-standard placement of that template.
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11403
Revision: 11403
Author: drtrigon
Date: 2013-04-21 09:27:06 +0000 (Sun, 21 Apr 2013)
Log Message:
-----------
improvement; further adoptions of logging to rewrite
(same output format and same code structure)
Modified Paths:
--------------
trunk/pywikipedia/wikipedia.py
Modified: trunk/pywikipedia/wikipedia.py
===================================================================
--- trunk/pywikipedia/wikipedia.py 2013-04-21 09:25:43 UTC (rev 11402)
+++ trunk/pywikipedia/wikipedia.py 2013-04-21 09:27:06 UTC (rev 11403)
@@ -9081,24 +9081,76 @@
logger.info(u'=== ' * 14)
+
+# Initialize the handlers and formatters for the logging system.
+#
+# ( Please confer branches/rewrite/pywikibot/bot.py for further info )
+
def setLogfileStatus(enabled, logname=None, header=False):
# NOTE-1: disable 'fh.setFormatter(formatter)' below in order to get "old"
# logging format (without additional info)
# NOTE-2: enable 'logger.addHandler(ch)' below in order output to console
# also (e.g. for simplifying 'pywikibot.output')
+ init_handlers(strm=None, logname=logname, header=header)
+ logger.propagate = enabled
+
+def init_handlers(strm=None, logname=None, header=False):
+ """Initialize logging system for terminal-based bots.
+
+ This function must be called before using pywikibot.output(); and must
+ be called again if the destination stream is changed.
+
+ @param strm: Output stream. If None, re-uses the last stream if one
+ was defined, otherwise uses sys.stderr
+
+ Note: this function is called by handleArgs(), so it should normally
+ not need to be called explicitly
+
+ All user output is routed through the logging module.
+ Each type of output is handled by an appropriate handler object.
+ This structure is used to permit eventual development of other
+ user interfaces (GUIs) without modifying the core bot code.
+ The following output levels are defined:
+ DEBUG - only for file logging; debugging messages
+ STDOUT - output that must be sent to sys.stdout (for bots that may
+ have their output redirected to a file or other destination)
+ VERBOSE - optional progress information for display to user
+ INFO - normal (non-optional) progress information for display to user
+ INPUT - prompts requiring user response
+ WARN - user warning messages
+ ERROR - user error messages
+ CRITICAL - fatal error messages
+ Accordingly, do ''not'' use print statements in bot code; instead,
+ use pywikibot.output function.
+ """
+
global logger
+
if not logger:
- if not logname:
- logname = '%s.log' % calledModuleName()
- if pywikibot.throttle.pid > 1:
- logname = '%s.%s.log' % (calledModuleName(), pywikibot.throttle.pid)
- logfn = config.datafilepath('logs', logname)
+ moduleName = calledModuleName()
+ if not moduleName:
+ moduleName = "terminal-interface"
+ logging.addLevelName(VERBOSE, "VERBOSE")
+ # for messages to be displayed on terminal at "verbose" setting
+ # use INFO for messages to be displayed even on non-verbose setting
+ logging.addLevelName(STDOUT, "STDOUT")
+ # for messages to be displayed to stdout
+ logging.addLevelName(INPUT, "INPUT")
+ # for prompts requiring user response
+
logger = logging.getLogger() # root logger
if logger.handlers: # init just once (if re-called)
logger = logging.getLogger('pywiki')
return
- logger.setLevel(INFO)
+ logger.setLevel(DEBUG+1) # all records except DEBUG go to logger
+
+ if not logname:
+ logname = '%s.log' % moduleName
+ if pywikibot.throttle.pid > 1:
+ logname = '%s.%s.log' % (moduleName, pywikibot.throttle.pid)
+ logfn = config.datafilepath('logs', logname)
+
# create file handler which logs even debug messages
if config.loghandler.upper() == 'RFH':
fh = logging.handlers.RotatingFileHandler(filename=logfn,
@@ -9127,9 +9179,8 @@
ch.setLevel(DEBUG)
# create formatter and add it to the handlers (using LogRecord attributes)
formatter = logging.Formatter(
- fmt='%(asctime)s %(name)18s: %(levelname)-8s %(message)s',
- #fmt="%(asctime)s %(filename)18s, %(lineno)4s "
- # "in %(funcName)18s: %(levelname)-8s %(message)s",
+ fmt="%(asctime)s %(caller_file)18s, %(caller_line)4s "
+ "in %(caller_name)18s: %(levelname)-8s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"
)
fh.setFormatter(formatter)
@@ -9143,8 +9194,6 @@
if header:
writeLogfileHeader()
- logger.propagate = enabled
-
writeToCommandLogFile()
colorTagR = re.compile('\03{.*?}', re.UNICODE)
@@ -9153,6 +9202,47 @@
input_lock = threading.Lock()
output_cache = []
+
+# User output/logging functions
+
+# Six output functions are defined. Each requires a unicode or string
+# argument. All of these functions generate a message to the log file if
+# logging is enabled ("-log" or "-debug" command line arguments).
+
+# The functions output(), stdout(), warning(), and error() all display a
+# message to the user through the logger object; the only difference is the
+# priority level, which can be used by the application layer to alter the
+# display. The stdout() function should be used only for data that is
+# the "result" of a script, as opposed to information messages to the
+# user.
+
+# The function log() by default does not display a message to the user, but
+# this can be altered by using the "-verbose" command line option.
+
+# The function debug() only logs its messages, they are never displayed on
+# the user console. debug() takes a required second argument, which is a
+# string indicating the debugging layer.
+
+# ( Please confer branches/rewrite/pywikibot/bot.py for further info )
+
+# next bit filched from 1.5.2's inspect.py
+def currentframe():
+ """Return the frame object for the caller's stack frame."""
+ try:
+ raise Exception
+ except:
+ # go back two levels, one for logoutput and one for whatever called it
+ return sys.exc_traceback.tb_frame.f_back.f_back
+
+if hasattr(sys, '_getframe'):
+ # less portable but more efficient
+ currentframe = lambda: sys._getframe(3)
+ # frame0 is this lambda, frame1 is logoutput() in this module,
+ # frame2 is the convenience function (output(), etc.)
+ # so frame3 is whatever called the convenience function
+
+# done filching
+
def logoutput(text, decoder=None, newline=True, _level=INFO, _logger="",
**kwargs):
"""Format output and send to the logging module.
@@ -9169,7 +9259,12 @@
if not logger:
setLogfileStatus(False)
- context = {}
+ frame = currentframe()
+ module = os.path.basename(frame.f_code.co_filename)
+ context = {'caller_name': frame.f_code.co_name,
+ 'caller_file': module,
+ 'caller_line': frame.f_lineno,
+ 'newline': ("\n" if newline else "")}
if decoder:
text = unicode(text, decoder)
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11402
Revision: 11402
Author: drtrigon
Date: 2013-04-21 09:25:43 +0000 (Sun, 21 Apr 2013)
Log Message:
-----------
minor docu change, important info into docstring instead of comment
Modified Paths:
--------------
branches/rewrite/pywikibot/bot.py
Modified: branches/rewrite/pywikibot/bot.py
===================================================================
--- branches/rewrite/pywikibot/bot.py 2013-04-20 20:45:30 UTC (rev 11401)
+++ branches/rewrite/pywikibot/bot.py 2013-04-21 09:25:43 UTC (rev 11402)
@@ -154,27 +154,27 @@
@param strm: Output stream. If None, re-uses the last stream if one
was defined, otherwise uses sys.stderr
+ Note: this function is called by handleArgs(), so it should normally
+ not need to be called explicitly
+
+ All user output is routed through the logging module.
+ Each type of output is handled by an appropriate handler object.
+ This structure is used to permit eventual development of other
+ user interfaces (GUIs) without modifying the core bot code.
+ The following output levels are defined:
+ DEBUG - only for file logging; debugging messages
+ STDOUT - output that must be sent to sys.stdout (for bots that may
+ have their output redirected to a file or other destination)
+ VERBOSE - optional progress information for display to user
+ INFO - normal (non-optional) progress information for display to user
+ INPUT - prompts requiring user response
+ WARN - user warning messages
+ ERROR - user error messages
+ CRITICAL - fatal error messages
+ Accordingly, do ''not'' use print statements in bot code; instead,
+ use pywikibot.output function.
"""
- # Note: this function is called by handleArgs(), so it should normally
- # not need to be called explicitly
- # All user output is routed through the logging module.
- # Each type of output is handled by an appropriate handler object.
- # This structure is used to permit eventual development of other
- # user interfaces (GUIs) without modifying the core bot code.
- # The following output levels are defined:
- # DEBUG - only for file logging; debugging messages
- # STDOUT - output that must be sent to sys.stdout (for bots that may
- # have their output redirected to a file or other destination)
- # VERBOSE - optional progress information for display to user
- # INFO - normal (non-optional) progress information for display to user
- # INPUT - prompts requiring user response
- # WARN - user warning messages
- # ERROR - user error messages
- # CRITICAL - fatal error messages
- # Accordingly, do ''not'' use print statements in bot code; instead,
- # use pywikibot.output function.
-
global _handlers_initialized
moduleName = calledModuleName()
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11397
Revision: 11397
Author: drtrigon
Date: 2013-04-20 12:20:45 +0000 (Sat, 20 Apr 2013)
Log Message:
-----------
bug fix; proper support of -debug by logging, follow-up to r11390 and r11395
Modified Paths:
--------------
trunk/pywikipedia/blockreview.py
trunk/pywikipedia/patrol.py
trunk/pywikipedia/sum_disc.py
trunk/pywikipedia/wikipedia.py
Modified: trunk/pywikipedia/blockreview.py
===================================================================
--- trunk/pywikipedia/blockreview.py 2013-04-20 09:11:14 UTC (rev 11396)
+++ trunk/pywikipedia/blockreview.py 2013-04-20 12:20:45 UTC (rev 11397)
@@ -140,7 +140,8 @@
talkComment = pywikibot.translate(self.site.lang, self.msg_user % self.parts)
# some test stuff
- if pywikibot.debug and self.site().loggedInAs() == u'Xqbot:':
+ if pywikibot.logger.isEnabledFor(pywikibot.DEBUG) \
+ and self.site().loggedInAs() == u'Xqbot:':
testPage = pywikibot.Page(self.site, 'Benutzer:Xqt/Test')
test = testPage.get()
test += note
Modified: trunk/pywikipedia/patrol.py
===================================================================
--- trunk/pywikipedia/patrol.py 2013-04-20 09:11:14 UTC (rev 11396)
+++ trunk/pywikipedia/patrol.py 2013-04-20 12:20:45 UTC (rev 11397)
@@ -150,8 +150,7 @@
# is registered as the user the rest of the structure
# refers to.
def process_children(obj,current_user):
- if pywikibot.debug:
- pywikibot.output(u'parsing node: %s' % obj)
+ pywikibot.debug(u'parsing node: %s' % obj)
for c in obj.children:
temp = process_node(c,current_user)
if temp and not current_user:
@@ -227,8 +226,7 @@
if author_ns:
author_ns_prefix = self.site.namespace(author_ns)
- if pywikibot.debug:
- pywikibot.output(u'Author ns: %d; name: %s' % (author_ns, author_ns_prefix))
+ pywikibot.debug(u'Author ns: %d; name: %s' % (author_ns, author_ns_prefix))
if title.find(author_ns_prefix+':') == 0:
return True
Modified: trunk/pywikipedia/sum_disc.py
===================================================================
--- trunk/pywikipedia/sum_disc.py 2013-04-20 09:11:14 UTC (rev 11396)
+++ trunk/pywikipedia/sum_disc.py 2013-04-20 12:20:45 UTC (rev 11397)
@@ -260,6 +260,7 @@
pywikibot.output(u'\03{lightgreen}* Initialization of bot:\03{default}')
+ # code debugging
logging.basicConfig(level=logging.DEBUG if ('code' in debug) else logging.INFO)
basic.AutoBasicBot.__init__(self)
@@ -315,9 +316,6 @@
# init variable/dynamic objects
- # code debugging
- pywikibot.debug = ('code' in debug)
-
## @todo re-write some functions to be pagegenerators and use pagegenerators.CombinedPageGenerator()
# and others to combine and use them
# \n[ JIRA: ticket? ]
Modified: trunk/pywikipedia/wikipedia.py
===================================================================
--- trunk/pywikipedia/wikipedia.py 2013-04-20 09:11:14 UTC (rev 11396)
+++ trunk/pywikipedia/wikipedia.py 2013-04-20 12:20:45 UTC (rev 11397)
@@ -5053,7 +5053,7 @@
def run(self):
if self.pages:
# Sometimes query does not contains revisions
- if self.site.has_api() and debug:
+ if self.site.has_api() and logger.isEnabledFor(DEBUG):
while True:
try:
data = self.getDataApi()
@@ -5441,8 +5441,10 @@
# TODO: why isn't this a Site method?
pages = list(pages) # if pages is an iterator, we need to make it a list
output(u'Getting %d page%s %sfrom %s...'
- % (len(pages), (u'', u's')[len(pages) != 1],
- (u'', u'via API ')[site.has_api() and debug], site))
+ %(len(pages),
+ (u'', u's')[len(pages) != 1],
+ (u'', u'via API ')[site.has_api() and logger.isEnabledFor(DEBUG)],
+ site))
limit = config.special_page_limit / 4 # default is 500/4, but It might have good point for server.
if len(pages) > limit:
# separate export pages for bulk-retrieve
@@ -8831,7 +8833,7 @@
args may be passed as an argument, thereby overriding sys.argv
"""
- global default_code, default_family, verbose, debug, simulate
+ global default_code, default_family, verbose, simulate
# get commandline arguments if necessary
if not args:
args = sys.argv[1:]
@@ -8888,7 +8890,9 @@
simulate = True
# global debug option for development purposes. Normally does nothing.
elif arg == '-debug':
- debug = True
+ if not logger:
+ setLogfileStatus(False)
+ logging.getLogger().setLevel(DEBUG)
config.special_page_limit = 500
else:
# the argument is not global. Let the specific bot script care
@@ -8999,7 +9003,6 @@
exec "import %s_interface as uiModule" % config.userinterface
ui = uiModule.UI()
verbose = 0
-debug = False
simulate = False
# TEST for bug #3081100
@@ -9084,7 +9087,7 @@
# NOTE-2: enable 'logger.addHandler(ch)' below in order output to console
# also (e.g. for simplifying 'pywikibot.output')
global logger
- if enabled:
+ if not logger:
if not logname:
logname = '%s.log' % calledModuleName()
if pywikibot.throttle.pid > 1:
@@ -9095,7 +9098,7 @@
if logger.handlers: # init just once (if re-called)
logger = logging.getLogger('pywiki')
return
- logger.setLevel(logging.DEBUG)
+ logger.setLevel(INFO)
# create file handler which logs even debug messages
if config.loghandler.upper() == 'RFH':
fh = logging.handlers.RotatingFileHandler(filename=logfn,
@@ -9118,10 +9121,10 @@
if os.path.exists(logfn) and (ver == int('0206')):
t = os.stat(logfn).st_mtime
fh.rolloverAt = fh.computeRollover(t)
- fh.setLevel(DEBUG if debug else INFO)
+ fh.setLevel(DEBUG)
# create console handler with a higher log level
ch = logging.StreamHandler()
- ch.setLevel(INFO)
+ ch.setLevel(DEBUG)
# create formatter and add it to the handlers (using LogRecord attributes)
formatter = logging.Formatter(
fmt='%(asctime)s %(name)18s: %(levelname)-8s %(message)s',
@@ -9139,12 +9142,9 @@
if header:
writeLogfileHeader()
- else:
- # disable the log file
- if logging.root: # resethandlers of root logger
- del logging.root.handlers[:]
- logger = logging.getLogger() # root logger
+ logger.propagate = enabled
+
writeToCommandLogFile()
colorTagR = re.compile('\03{.*?}', re.UNICODE)
@@ -9190,7 +9190,8 @@
# instead of logging handler for output to console (StreamHandler)
if _level <> INFO:
text = u'%s: %s' % (logging.getLevelName(_level), text)
- _outputOld(text, decoder, newline, (_level == STDOUT), **kwargs)
+ if log.isEnabledFor(_level):
+ _outputOld(text, decoder, newline, (_level == STDOUT), **kwargs)
def _outputOld(text, decoder=None, newline=True, toStdout=False, **kwargs):
"""Output a message to the user via the userinterface.