http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11412
Revision: 11412
Author: xqt
Date: 2013-04-21 13:07:03 +0000 (Sun, 21 Apr 2013)
Log Message:
-----------
revert r11399 since the bug is solved in r11406
Modified Paths:
--------------
trunk/pywikipedia/cosmetic_changes.py
Modified: trunk/pywikipedia/cosmetic_changes.py
===================================================================
--- trunk/pywikipedia/cosmetic_changes.py 2013-04-21 13:04:36 UTC (rev 11411)
+++ trunk/pywikipedia/cosmetic_changes.py 2013-04-21 13:07:03 UTC (rev 11412)
@@ -159,13 +159,13 @@
text = self.fixSyntaxSave(text)
text = self.cleanUpLinks(text)
text = self.cleanUpSectionHeaders(text)
-## text = self.putSpacesInLists(text)
+ text = self.putSpacesInLists(text)
text = self.translateAndCapitalizeNamespaces(text)
## text = self.translateMagicWords(text)
text = self.replaceDeprecatedTemplates(text)
## text = self.resolveHtmlEntities(text)
text = self.validXhtml(text)
-## text = self.removeUselessSpaces(text)
+ text = self.removeUselessSpaces(text)
text = self.removeNonBreakingSpaceBeforePercent(text)
text = self.fixHtml(text)
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11406
Revision: 11406
Author: xqt
Date: 2013-04-21 12:03:57 +0000 (Sun, 21 Apr 2013)
Log Message:
-----------
- Replace marked nested templates in reverse order, fix for bug #3611198;
- speed up replacement for duplicate templates
- bugfix for pywikipedia.Error
Modified Paths:
--------------
trunk/pywikipedia/pywikibot/textlib.py
Modified: trunk/pywikipedia/pywikibot/textlib.py
===================================================================
--- trunk/pywikipedia/pywikibot/textlib.py 2013-04-21 10:55:57 UTC (rev 11405)
+++ trunk/pywikipedia/pywikibot/textlib.py 2013-04-21 12:03:57 UTC (rev 11406)
@@ -140,16 +140,23 @@
text = text.replace(item, '%s%d%s' % (marker2, count, marker2))
values[count] = item
inside = {}
- count = 0
+ seen = set()
while TEMP_REGEX.search(text) is not None:
for m in TEMP_REGEX.finditer(text):
- count += 1
item = m.group()
+ if item in seen:
+ continue # speed up
+ seen.add(item)
+ count = len(seen)
text = text.replace(item, '%s%d%s' % (marker1, count, marker1))
# Make sure stored templates don't contain markers
- for m2 in Rmarker1.finditer(item):
- item = item.replace(m2.group(), inside[int(m2.group(1))])
+ # We replace the last item first, otherwise inside templates
+ # like {{A{{B}}{{C}}1{{D}}}} could fail
+ for i in range(count - 1, 0, -1):
+ item = item.replace(u'%(mark)s%(number)s%(mark)s'
+ % {'mark': marker1, 'number': i},
+ inside[i])
for m2 in Rmarker2.finditer(item):
item = item.replace(m2.group(), values[int(m2.group(1))])
inside[count] = item
@@ -745,7 +752,7 @@
if site is None:
site = pywikibot.getSite()
if site.sitename() == 'wikipedia:de' and "{{Personendaten" in oldtext:
- raise Error("""\
+ raise pywikibot.Error("""\
The PyWikipediaBot is no longer allowed to touch categories on the German
Wikipedia on pages that contain the Personendaten template because of the
non-standard placement of that template.
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11403
Revision: 11403
Author: drtrigon
Date: 2013-04-21 09:27:06 +0000 (Sun, 21 Apr 2013)
Log Message:
-----------
improvement; further adoptions of logging to rewrite
(same output format and same code structure)
Modified Paths:
--------------
trunk/pywikipedia/wikipedia.py
Modified: trunk/pywikipedia/wikipedia.py
===================================================================
--- trunk/pywikipedia/wikipedia.py 2013-04-21 09:25:43 UTC (rev 11402)
+++ trunk/pywikipedia/wikipedia.py 2013-04-21 09:27:06 UTC (rev 11403)
@@ -9081,24 +9081,76 @@
logger.info(u'=== ' * 14)
+
+# Initialize the handlers and formatters for the logging system.
+#
+# ( Please confer branches/rewrite/pywikibot/bot.py for further info )
+
def setLogfileStatus(enabled, logname=None, header=False):
# NOTE-1: disable 'fh.setFormatter(formatter)' below in order to get "old"
# logging format (without additional info)
# NOTE-2: enable 'logger.addHandler(ch)' below in order output to console
# also (e.g. for simplifying 'pywikibot.output')
+ init_handlers(strm=None, logname=logname, header=header)
+ logger.propagate = enabled
+
+def init_handlers(strm=None, logname=None, header=False):
+ """Initialize logging system for terminal-based bots.
+
+ This function must be called before using pywikibot.output(); and must
+ be called again if the destination stream is changed.
+
+ @param strm: Output stream. If None, re-uses the last stream if one
+ was defined, otherwise uses sys.stderr
+
+ Note: this function is called by handleArgs(), so it should normally
+ not need to be called explicitly
+
+ All user output is routed through the logging module.
+ Each type of output is handled by an appropriate handler object.
+ This structure is used to permit eventual development of other
+ user interfaces (GUIs) without modifying the core bot code.
+ The following output levels are defined:
+ DEBUG - only for file logging; debugging messages
+ STDOUT - output that must be sent to sys.stdout (for bots that may
+ have their output redirected to a file or other destination)
+ VERBOSE - optional progress information for display to user
+ INFO - normal (non-optional) progress information for display to user
+ INPUT - prompts requiring user response
+ WARN - user warning messages
+ ERROR - user error messages
+ CRITICAL - fatal error messages
+ Accordingly, do ''not'' use print statements in bot code; instead,
+ use pywikibot.output function.
+ """
+
global logger
+
if not logger:
- if not logname:
- logname = '%s.log' % calledModuleName()
- if pywikibot.throttle.pid > 1:
- logname = '%s.%s.log' % (calledModuleName(), pywikibot.throttle.pid)
- logfn = config.datafilepath('logs', logname)
+ moduleName = calledModuleName()
+ if not moduleName:
+ moduleName = "terminal-interface"
+ logging.addLevelName(VERBOSE, "VERBOSE")
+ # for messages to be displayed on terminal at "verbose" setting
+ # use INFO for messages to be displayed even on non-verbose setting
+ logging.addLevelName(STDOUT, "STDOUT")
+ # for messages to be displayed to stdout
+ logging.addLevelName(INPUT, "INPUT")
+ # for prompts requiring user response
+
logger = logging.getLogger() # root logger
if logger.handlers: # init just once (if re-called)
logger = logging.getLogger('pywiki')
return
- logger.setLevel(INFO)
+ logger.setLevel(DEBUG+1) # all records except DEBUG go to logger
+
+ if not logname:
+ logname = '%s.log' % moduleName
+ if pywikibot.throttle.pid > 1:
+ logname = '%s.%s.log' % (moduleName, pywikibot.throttle.pid)
+ logfn = config.datafilepath('logs', logname)
+
# create file handler which logs even debug messages
if config.loghandler.upper() == 'RFH':
fh = logging.handlers.RotatingFileHandler(filename=logfn,
@@ -9127,9 +9179,8 @@
ch.setLevel(DEBUG)
# create formatter and add it to the handlers (using LogRecord attributes)
formatter = logging.Formatter(
- fmt='%(asctime)s %(name)18s: %(levelname)-8s %(message)s',
- #fmt="%(asctime)s %(filename)18s, %(lineno)4s "
- # "in %(funcName)18s: %(levelname)-8s %(message)s",
+ fmt="%(asctime)s %(caller_file)18s, %(caller_line)4s "
+ "in %(caller_name)18s: %(levelname)-8s %(message)s",
datefmt="%Y-%m-%d %H:%M:%S"
)
fh.setFormatter(formatter)
@@ -9143,8 +9194,6 @@
if header:
writeLogfileHeader()
- logger.propagate = enabled
-
writeToCommandLogFile()
colorTagR = re.compile('\03{.*?}', re.UNICODE)
@@ -9153,6 +9202,47 @@
input_lock = threading.Lock()
output_cache = []
+
+# User output/logging functions
+
+# Six output functions are defined. Each requires a unicode or string
+# argument. All of these functions generate a message to the log file if
+# logging is enabled ("-log" or "-debug" command line arguments).
+
+# The functions output(), stdout(), warning(), and error() all display a
+# message to the user through the logger object; the only difference is the
+# priority level, which can be used by the application layer to alter the
+# display. The stdout() function should be used only for data that is
+# the "result" of a script, as opposed to information messages to the
+# user.
+
+# The function log() by default does not display a message to the user, but
+# this can be altered by using the "-verbose" command line option.
+
+# The function debug() only logs its messages, they are never displayed on
+# the user console. debug() takes a required second argument, which is a
+# string indicating the debugging layer.
+
+# ( Please confer branches/rewrite/pywikibot/bot.py for further info )
+
+# next bit filched from 1.5.2's inspect.py
+def currentframe():
+ """Return the frame object for the caller's stack frame."""
+ try:
+ raise Exception
+ except:
+ # go back two levels, one for logoutput and one for whatever called it
+ return sys.exc_traceback.tb_frame.f_back.f_back
+
+if hasattr(sys, '_getframe'):
+ # less portable but more efficient
+ currentframe = lambda: sys._getframe(3)
+ # frame0 is this lambda, frame1 is logoutput() in this module,
+ # frame2 is the convenience function (output(), etc.)
+ # so frame3 is whatever called the convenience function
+
+# done filching
+
def logoutput(text, decoder=None, newline=True, _level=INFO, _logger="",
**kwargs):
"""Format output and send to the logging module.
@@ -9169,7 +9259,12 @@
if not logger:
setLogfileStatus(False)
- context = {}
+ frame = currentframe()
+ module = os.path.basename(frame.f_code.co_filename)
+ context = {'caller_name': frame.f_code.co_name,
+ 'caller_file': module,
+ 'caller_line': frame.f_lineno,
+ 'newline': ("\n" if newline else "")}
if decoder:
text = unicode(text, decoder)