http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11490
Revision: 11490
Author: drtrigon
Date: 2013-05-03 22:36:34 +0000 (Fri, 03 May 2013)
Log Message:
-----------
bug fix; for issue introduced in r11387 and with this a partly revert of r11474
this bug fix avoids the recursion error and introduces a check when to output the header by checking whether the site object is available
Modified Paths:
--------------
branches/rewrite/pywikibot/bot.py
Modified: branches/rewrite/pywikibot/bot.py
===================================================================
--- branches/rewrite/pywikibot/bot.py 2013-05-03 21:17:14 UTC (rev 11489)
+++ branches/rewrite/pywikibot/bot.py 2013-05-03 22:36:34 UTC (rev 11490)
@@ -226,23 +226,28 @@
debuglogger.setLevel(DEBUG)
debuglogger.addHandler(file_handler)
-## writelogheader()
-
_handlers_initialized = True
+ writelogheader()
+
def writelogheader():
"""
Save additional version, system and status info to the logfile in use,
so that the user can look it up later to track errors or report bugs.
"""
+ # if site not available it's too early to print a header (work-a-round)
+ try:
+ site = pywikibot.getSite()
+ except:
+ return
+
output(u'=== Pywikipediabot framework v2.0 -- Logging header ===')
# script call
output(u'COMMAND: %s' % unicode(sys.argv))
# new framework release/revision? (handleArgs needs to be called first)
- site = pywikibot.getSite()
output(u'VERSION: %s' % unicode((version.getversion().strip(),
version.getversion_onlinerepo(),
site.live_version())))
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11489
Revision: 11489
Author: drtrigon
Date: 2013-05-03 21:17:14 +0000 (Fri, 03 May 2013)
Log Message:
-----------
bug fix; for bug #3611511 introduced in r11397
undo; some changes from r11479 and r11480 since the development switch 'debug' here was NOT removed with r11397 BUT REPLACED with 'pywikibot.logger.isEnabledFor(DEBUG)' (which is a bit unhandy - I agree...)
Modified Paths:
--------------
trunk/pywikipedia/blockreview.py
trunk/pywikipedia/wikipedia.py
Modified: trunk/pywikipedia/blockreview.py
===================================================================
--- trunk/pywikipedia/blockreview.py 2013-05-03 08:14:51 UTC (rev 11488)
+++ trunk/pywikipedia/blockreview.py 2013-05-03 21:17:14 UTC (rev 11489)
@@ -145,7 +145,8 @@
% self.parts)
# some test stuff
- if True and self.site().loggedInAs() == u'Xqbot:':
+ if pywikibot.logger.isEnabledFor(pywikibot.DEBUG) \
+ and self.site().loggedInAs() == u'Xqbot:':
testPage = pywikibot.Page(self.site,
'Benutzer:Xqt/Test')
test = testPage.get()
Modified: trunk/pywikipedia/wikipedia.py
===================================================================
--- trunk/pywikipedia/wikipedia.py 2013-05-03 08:14:51 UTC (rev 11488)
+++ trunk/pywikipedia/wikipedia.py 2013-05-03 21:17:14 UTC (rev 11489)
@@ -5062,7 +5062,7 @@
# Sometimes query does not contains revisions
# or some pages are missing. Deactivate api call and use the
# old API special:export
- if self.site.has_api() and False:
+ if self.site.has_api() and logger.isEnabledFor(DEBUG):
while True:
try:
data = self.getDataApi()
@@ -5450,7 +5450,7 @@
{'count': len(pages),
# API is deactivated since r8036 because some pages are missing
'API': (u'',
- u'via API ')[site.has_api() and False],
+ u'via API ')[site.has_api() and logger.isEnabledFor(DEBUG)],
'site': site}))
limit = config.special_page_limit / 4 # default is 500/4, but It might have good point for server.
if len(pages) > limit:
@@ -8898,7 +8898,7 @@
# global debug option for development purposes. Normally does nothing.
elif arg == '-debug':
if not logger:
- setLogfileStatus(False)
+ init_handlers()
logging.getLogger().setLevel(DEBUG)
config.special_page_limit = 500
else:
@@ -8909,6 +8909,8 @@
if username:
config.usernames[default_family][default_code] = username
+ #init_handlers()
+
# TEST for bug #3081100
if unicode_error:
output("""
@@ -9022,7 +9024,6 @@
default_family = config.family
default_code = config.mylang
-logger = None
# Check
# if the default family+wiki is a non-public one,
@@ -9061,67 +9062,24 @@
#
# ( Please confer branches/rewrite/pywikibot/bot.py for further info )
+logger = None
+#_handlers_initialized = False
+
def setLogfileStatus(enabled, logname=None, header=False):
# NOTE-1: disable 'fh.setFormatter(formatter)' below in order to get "old"
# logging format (without additional info)
# NOTE-2: enable 'logger.addHandler(ch)' below in order output to console
# also (e.g. for simplifying 'pywikibot.output')
- init_handlers(strm=None, logname=logname, header=header)
- logger.propagate = enabled
+ global logger #_handlers_initialized
-def init_handlers(strm=None, logname=None, header=False):
- """Initialize logging system for terminal-based bots.
+ if not logger:
+ init_handlers()
- This function must be called before using pywikibot.output(); and must
- be called again if the destination stream is changed.
-
- @param strm: Output stream. If None, re-uses the last stream if one
- was defined, otherwise uses sys.stderr
-
- Note: this function is called by handleArgs(), so it should normally
- not need to be called explicitly
-
- All user output is routed through the logging module.
- Each type of output is handled by an appropriate handler object.
- This structure is used to permit eventual development of other
- user interfaces (GUIs) without modifying the core bot code.
- The following output levels are defined:
- DEBUG - only for file logging; debugging messages
- STDOUT - output that must be sent to sys.stdout (for bots that may
- have their output redirected to a file or other destination)
- VERBOSE - optional progress information for display to user
- INFO - normal (non-optional) progress information for display to user
- INPUT - prompts requiring user response
- WARN - user warning messages
- ERROR - user error messages
- CRITICAL - fatal error messages
- Accordingly, do ''not'' use print statements in bot code; instead,
- use pywikibot.output function.
- """
-
- global logger
-
- if not logger:
+ if not logger.handlers: # init just once (if re-called)
moduleName = calledModuleName()
if not moduleName:
moduleName = "terminal-interface"
- logging.addLevelName(VERBOSE, "VERBOSE")
- # for messages to be displayed on terminal at "verbose" setting
- # use INFO for messages to be displayed even on non-verbose setting
- logging.addLevelName(STDOUT, "STDOUT")
- # for messages to be displayed to stdout
- logging.addLevelName(INPUT, "INPUT")
- # for prompts requiring user response
-
- logger = logging.getLogger() # root logger
- if logger.handlers: # init just once (if re-called)
- logger = logging.getLogger('pywiki')
- return
- logger.setLevel(DEBUG+1) # all records except DEBUG go to logger
- if hasattr(logger, 'captureWarnings'):
- logger.captureWarnings(True) # introduced in Python >= 2.7
-
if not logname:
logname = '%s.log' % moduleName
if pywikibot.throttle.pid > 1:
@@ -9171,6 +9129,59 @@
if header:
writelogheader()
+ logger.propagate = enabled
+
+def init_handlers(strm=None):#, logname=None, header=False):
+ """Initialize logging system for terminal-based bots.
+
+ This function must be called before using pywikibot.output(); and must
+ be called again if the destination stream is changed.
+
+ @param strm: Output stream. If None, re-uses the last stream if one
+ was defined, otherwise uses sys.stderr
+
+ Note: this function is called by handleArgs(), so it should normally
+ not need to be called explicitly
+
+ All user output is routed through the logging module.
+ Each type of output is handled by an appropriate handler object.
+ This structure is used to permit eventual development of other
+ user interfaces (GUIs) without modifying the core bot code.
+ The following output levels are defined:
+ DEBUG - only for file logging; debugging messages
+ STDOUT - output that must be sent to sys.stdout (for bots that may
+ have their output redirected to a file or other destination)
+ VERBOSE - optional progress information for display to user
+ INFO - normal (non-optional) progress information for display to user
+ INPUT - prompts requiring user response
+ WARN - user warning messages
+ ERROR - user error messages
+ CRITICAL - fatal error messages
+ Accordingly, do ''not'' use print statements in bot code; instead,
+ use pywikibot.output function.
+ """
+ # currently only the logger is initialized here
+ # the handlers are initialized in setLogfileStatus
+
+ global logger #_handlers_initialized
+
+ if not logger:
+ logging.addLevelName(VERBOSE, "VERBOSE")
+ # for messages to be displayed on terminal at "verbose" setting
+ # use INFO for messages to be displayed even on non-verbose setting
+ logging.addLevelName(STDOUT, "STDOUT")
+ # for messages to be displayed to stdout
+ logging.addLevelName(INPUT, "INPUT")
+ # for prompts requiring user response
+
+ logger = logging.getLogger() # root logger
+ if logger.handlers: # init just once (if re-called)
+ logger = logging.getLogger('pywiki')
+ return
+ logger.setLevel(DEBUG+1) # all records except DEBUG go to logger
+ if hasattr(logger, 'captureWarnings'):
+ logger.captureWarnings(True) # introduced in Python >= 2.7
+
def writelogheader():
"""
Save additional version, system and status info to the logfile in use,
@@ -9203,6 +9214,8 @@
output(u'=== ' * 14)
+init_handlers() # guarantee 'logger' to be defined
+
writeToCommandLogFile()
colorTagR = re.compile('\03{.*?}', re.UNICODE)
@@ -9266,7 +9279,7 @@
# make sure logging system has been initialized
if not logger:
- setLogfileStatus(False)
+ init_handlers()
frame = currentframe()
module = os.path.basename(frame.f_code.co_filename)
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11484
Revision: 11484
Author: xqt
Date: 2013-04-28 17:09:55 +0000 (Sun, 28 Apr 2013)
Log Message:
-----------
If we have digits between brackets like {{foo}}1{{bar}}, restoring from dict may fail. We need to change the index.
Follow up for r11410 update from trunk r11483.
Modified Paths:
--------------
branches/rewrite/pywikibot/textlib.py
Modified: branches/rewrite/pywikibot/textlib.py
===================================================================
--- branches/rewrite/pywikibot/textlib.py 2013-04-28 17:04:21 UTC (rev 11483)
+++ branches/rewrite/pywikibot/textlib.py 2013-04-28 17:09:55 UTC (rev 11484)
@@ -133,30 +133,35 @@
Rmarker2 = re.compile('%(mark)s(\d+)%(mark)s' % {'mark': marker2})
# hide the flat template marker
dontTouchRegexes.append(Rmarker1)
+ origin = text
values = {}
count = 0
for m in Rvalue.finditer(text):
count += 1
+ # If we have digits between brackets, restoring from dict may fail.
+ # So we need to change the index. We have to search in the origin.
+ while u'}}}%d{{{' % count in origin:
+ count += 1
item = m.group()
text = text.replace(item, '%s%d%s' % (marker2, count, marker2))
values[count] = item
inside = {}
seen = set()
+ count = 0
while TEMP_REGEX.search(text) is not None:
for m in TEMP_REGEX.finditer(text):
item = m.group()
if item in seen:
continue # speed up
seen.add(item)
- count = len(seen)
+ count += 1
+ while u'}}%d{{' % count in origin:
+ count += 1
text = text.replace(item, '%s%d%s' % (marker1, count, marker1))
# Make sure stored templates don't contain markers
- # We replace the last item first, otherwise inside templates
- # like {{A{{B}}{{C}}1{{D}}}} could fail
- for i in range(count - 1, 0, -1):
- item = item.replace('%s%d%s' % (marker1, i, marker1),
- inside[i])
+ for m2 in Rmarker1.finditer(item):
+ item = item.replace(m2.group(), inside[int(m2.group(1))])
for m2 in Rmarker2.finditer(item):
item = item.replace(m2.group(), values[int(m2.group(1))])
inside[count] = item
@@ -889,7 +894,7 @@
thistxt = removeDisabledParts(text)
# marker for inside templates or parameters
- marker = findmarker(thistxt)
+ marker1 = findmarker(thistxt)
# marker for links
marker2 = findmarker(thistxt, u'##', u'#')
@@ -903,7 +908,7 @@
result = []
Rmath = re.compile(ur'<math>[^<]+</math>')
Rvalue = re.compile(r'{{{.+?}}}')
- Rmarker = re.compile(ur'%s(\d+)%s' % (marker, marker))
+ Rmarker1 = re.compile(ur'%s(\d+)%s' % (marker1, marker1))
Rmarker2 = re.compile(ur'%s(\d+)%s' % (marker2, marker2))
Rmarker3 = re.compile(ur'%s(\d+)%s' % (marker3, marker3))
Rmarker4 = re.compile(ur'%s(\d+)%s' % (marker4, marker4))
@@ -921,12 +926,17 @@
count = 0
for m in Rvalue.finditer(thistxt):
count += 1
+ # If we have digits between brackets, restoring from dict may fail.
+ # So we need to change the index. We have to search in the origin text.
+ while u'}}}%d{{{' % count in text:
+ count += 1
item = m.group()
thistxt = thistxt.replace(item, '%s%d%s' % (marker4, count, marker4))
values[count] = item
inside = {}
seen = set()
+ count = 0
while TEMP_REGEX.search(thistxt) is not None:
for m in TEMP_REGEX.finditer(thistxt):
# Make sure it is not detected again
@@ -934,14 +944,15 @@
if item in seen:
continue # speed up
seen.add(item)
- count = len(seen)
- thistxt = thistxt.replace(item, '%s%d%s' % (marker, count, marker))
+ count += 1
+ while u'}}%d{{' % count in text:
+ count += 1
+ thistxt = thistxt.replace(item,
+ '%s%d%s' % (marker1, count, marker1))
+
# Make sure stored templates don't contain markers
- # We replace the last item first, otherwise inside templates
- # like {{A|{{B}}{{C}}1{{D}}}} could fail
- for i in range(count - 1, 0, -1):
- item = item.replace('%s%d%s' % (marker, count, marker),
- inside[i])
+ for m2 in Rmarker1.finditer(item):
+ item = item.replace(m2.group(), inside[int(m2.group(1))])
for m2 in Rmarker3.finditer(item):
item = item.replace(m2.group(), maths[int(m2.group(1))])
for m2 in Rmarker4.finditer(item):
@@ -950,7 +961,7 @@
# Name
name = m.group('name').strip()
- m2 = Rmarker.search(name) or Rmath.search(name)
+ m2 = Rmarker1.search(name) or Rmath.search(name)
if m2 is not None:
# Doesn't detect templates whose name changes,
# or templates whose name contains math tags
@@ -1009,10 +1020,9 @@
param_val = param
numbered_param += 1
count = len(inside)
- for i in range(count - 1, 0, -1):
- param_val = param_val.replace('%s%d%s'
- % (marker, i, marker),
- inside[i])
+ for m2 in Rmarker1.finditer(param_val):
+ param_val = param_val.replace(m2.group(),
+ inside[int(m2.group(1))])
for m2 in Rmarker2.finditer(param_val):
param_val = param_val.replace(m2.group(),
links[int(m2.group(1))])
http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11483
Revision: 11483
Author: xqt
Date: 2013-04-28 17:04:21 +0000 (Sun, 28 Apr 2013)
Log Message:
-----------
If we have digits between brackets like {{foo}}1{{bar}}, restoring from dict may fail. We need to change the index. Follow up for r11406, r11408.
Modified Paths:
--------------
trunk/pywikipedia/pywikibot/textlib.py
Modified: trunk/pywikipedia/pywikibot/textlib.py
===================================================================
--- trunk/pywikipedia/pywikibot/textlib.py 2013-04-28 13:23:57 UTC (rev 11482)
+++ trunk/pywikipedia/pywikibot/textlib.py 2013-04-28 17:04:21 UTC (rev 11483)
@@ -133,30 +133,35 @@
Rmarker2 = re.compile('%(mark)s(\d+)%(mark)s' % {'mark': marker2})
# hide the flat template marker
dontTouchRegexes.append(Rmarker1)
+ origin = text
values = {}
count = 0
for m in Rvalue.finditer(text):
count += 1
+ # If we have digits between brackets, restoring from dict may fail.
+ # So we need to change the index. We have to search in the origin.
+ while u'}}}%d{{{' % count in origin:
+ count += 1
item = m.group()
text = text.replace(item, '%s%d%s' % (marker2, count, marker2))
values[count] = item
inside = {}
seen = set()
+ count = 0
while TEMP_REGEX.search(text) is not None:
for m in TEMP_REGEX.finditer(text):
item = m.group()
if item in seen:
continue # speed up
seen.add(item)
- count = len(seen)
+ count += 1
+ while u'}}%d{{' % count in origin:
+ count += 1
text = text.replace(item, '%s%d%s' % (marker1, count, marker1))
# Make sure stored templates don't contain markers
- # We replace the last item first, otherwise inside templates
- # like {{A{{B}}{{C}}1{{D}}}} could fail
- for i in range(count - 1, 0, -1):
- item = item.replace('%s%d%s' % (marker1, i, marker1),
- inside[i])
+ for m2 in Rmarker1.finditer(item):
+ item = item.replace(m2.group(), inside[int(m2.group(1))])
for m2 in Rmarker2.finditer(item):
item = item.replace(m2.group(), values[int(m2.group(1))])
inside[count] = item
@@ -899,7 +904,7 @@
thistxt = removeDisabledParts(text)
# marker for inside templates or parameters
- marker = findmarker(thistxt)
+ marker1 = findmarker(thistxt)
# marker for links
marker2 = findmarker(thistxt, u'##', u'#')
@@ -913,7 +918,7 @@
result = []
Rmath = re.compile(ur'<math>[^<]+</math>')
Rvalue = re.compile(r'{{{.+?}}}')
- Rmarker = re.compile(ur'%s(\d+)%s' % (marker, marker))
+ Rmarker1 = re.compile(ur'%s(\d+)%s' % (marker1, marker1))
Rmarker2 = re.compile(ur'%s(\d+)%s' % (marker2, marker2))
Rmarker3 = re.compile(ur'%s(\d+)%s' % (marker3, marker3))
Rmarker4 = re.compile(ur'%s(\d+)%s' % (marker4, marker4))
@@ -931,12 +936,17 @@
count = 0
for m in Rvalue.finditer(thistxt):
count += 1
+ # If we have digits between brackets, restoring from dict may fail.
+ # So we need to change the index. We have to search in the origin text.
+ while u'}}}%d{{{' % count in text:
+ count += 1
item = m.group()
thistxt = thistxt.replace(item, '%s%d%s' % (marker4, count, marker4))
values[count] = item
inside = {}
seen = set()
+ count = 0
while TEMP_REGEX.search(thistxt) is not None:
for m in TEMP_REGEX.finditer(thistxt):
# Make sure it is not detected again
@@ -944,14 +954,15 @@
if item in seen:
continue # speed up
seen.add(item)
- count = len(seen)
- thistxt = thistxt.replace(item, '%s%d%s' % (marker, count, marker))
+ count += 1
+ while u'}}%d{{' % count in text:
+ count += 1
+ thistxt = thistxt.replace(item,
+ '%s%d%s' % (marker1, count, marker1))
+
# Make sure stored templates don't contain markers
- # We replace the last item first, otherwise inside templates
- # like {{A|{{B}}{{C}}1{{D}}}} could fail
- for i in range(count - 1, 0, -1):
- item = item.replace('%s%d%s' % (marker, count, marker),
- inside[i])
+ for m2 in Rmarker1.finditer(item):
+ item = item.replace(m2.group(), inside[int(m2.group(1))])
for m2 in Rmarker3.finditer(item):
item = item.replace(m2.group(), maths[int(m2.group(1))])
for m2 in Rmarker4.finditer(item):
@@ -960,7 +971,7 @@
# Name
name = m.group('name').strip()
- m2 = Rmarker.search(name) or Rmath.search(name)
+ m2 = Rmarker1.search(name) or Rmath.search(name)
if m2 is not None:
# Doesn't detect templates whose name changes,
# or templates whose name contains math tags
@@ -1019,10 +1030,9 @@
param_val = param
numbered_param += 1
count = len(inside)
- for i in range(count - 1, 0, -1):
- param_val = param_val.replace('%s%d%s'
- % (marker, i, marker),
- inside[i])
+ for m2 in Rmarker1.finditer(param_val):
+ param_val = param_val.replace(m2.group(),
+ inside[int(m2.group(1))])
for m2 in Rmarker2.finditer(param_val):
param_val = param_val.replace(m2.group(),
links[int(m2.group(1))])