http://www.mediawiki.org/wiki/Special:Code/pywikipedia/11371
Revision: 11371 Author: drtrigon Date: 2013-04-16 16:37:10 +0000 (Tue, 16 Apr 2013) Log Message: ----------- improvements; code cleanup, config adoptions, close to productive for wikidata follow-up; to r11327 and r11328
Modified Paths: -------------- trunk/pywikipedia/subster.py trunk/pywikipedia/subster_irc.py
Modified: trunk/pywikipedia/subster.py =================================================================== --- trunk/pywikipedia/subster.py 2013-04-16 03:09:20 UTC (rev 11370) +++ trunk/pywikipedia/subster.py 2013-04-16 16:37:10 UTC (rev 11371) @@ -85,10 +85,16 @@
bot_config = { # unicode values + 'BotName': pywikibot.config.usernames[pywikibot.config.family][pywikibot.config.mylang], 'TemplateName': u'User:DrTrigonBot/Subster', # or 'template' for 'Flagged Revisions' + 'data_PropertyId': u'370', # default: Sandbox-String (P370)
'ErrorTemplate': u'<b>SubsterBot Exception in "%s" (%s)</b>\n<pre>%s</pre>', 'VerboseMessage': u'<noinclude>\n----\n%s\n</noinclude>', # DRTRIGON-116, DRTRIGON-132 + 'data_VerboseMessage': u'<onlyinclude>{{#switch: {{{1|}}}\n' + u'| error = %(error)s\n' + u'|error-traceback = %(error-traceback)s\n' + u'}}</onlyinclude>',
# important to use a '.css' page here, since it HAS TO BE protected to # prevent malicious code injection ! @@ -142,13 +148,11 @@ other wiki text data. Like dynamic text updating. '''
- _param_default = bot_config['param_default'] - _var_regex_str = bot_config['var_regex_str']%{'var1':'%(var)s','var2':'%(var)s','cont':'%(cont)s'} _BS_regex_str = bot_config['var_regex_str']%{'var1':'%(var1)s','var2':'%(var2)sBS:/','cont':'%(cont)s'}
# -template and subst-tag handling taken from MerlBot - # -this bot could also be runned on my local wiki with an anacron-job + # -this bot could also be runned on any local wiki with an anacron-job
def __init__(self): '''Constructor of SubsterBot(), initialize needed vars.''' @@ -173,13 +177,15 @@ # convert e.g. namespaces to corret language self._bot_config['TemplateName'] = pywikibot.Page(self.site, self._bot_config['TemplateName']).title() self._template_regex = re.compile('{{' + self._bot_config['TemplateName'] + '(.*?)}}', re.S) + if self.site.is_data_repository(): + self._bot_config['VerboseMessage'] = self._bot_config['data_VerboseMessage']
self._debug = debug
# init constants - self._userListPage = pywikibot.Page(self.site, bot_config['TemplateName']) - self._ConfCSSpostprocPage = pywikibot.Page(self.site, bot_config['ConfCSSpostproc']) - self._ConfCSSconfigPage = pywikibot.Page(self.site, bot_config['ConfCSSconfig']) + self._userListPage = pywikibot.Page(self.site, self._bot_config['TemplateName']) + self._ConfCSSpostprocPage = pywikibot.Page(self.site, self._bot_config['ConfCSSpostproc']) + self._ConfCSSconfigPage = pywikibot.Page(self.site, self._bot_config['ConfCSSconfig']) self.pagegen = pagegenerators.ReferringPageGenerator(self._userListPage, onlyTemplateInclusion=True) self._code = self._ConfCSSpostprocPage.get() pywikibot.output(u'Imported postproc %s rev %s from %s' %\ @@ -210,7 +216,7 @@ # get page content and operating mode content = self.load(page) params = self.loadTemplates(page, self._bot_config['TemplateName'], - default=self._param_default) + default=self._bot_config['param_default'])
if not params: continue
@@ -241,7 +247,6 @@ # DRTRIGON-130: data repository (wikidata) output to items if self.site.is_data_repository(): data = self.data_convertContent(substed_content) - #print self.data_save(dataoutpage, dic[u'claims'], {u'p32': data}, summary) self.data_save(page, data) else: pywikibot.output(u'NOTHING TO DO!') @@ -285,7 +290,7 @@ # DRTRIGON-132; metadata append IFF exception raised # (this metadata HAVE TO trigger a change because of error!) metadata['bot-error'] = unicode(True) - metadata['bot-error-traceback'] = bot_config['ErrorTemplate'] %\ + metadata['bot-error-traceback'] = self._bot_config['ErrorTemplate'] %\ ( item['value'], pywikibot.Timestamp.now().isoformat(' '), result.strip() ) @@ -295,7 +300,7 @@ value = md_val_tag % (item['value'], 'bot-error-traceback') tags = self.subTag(substed_content, value)[1] if ast.literal_eval(item['verbose']) and (value not in tags): - substed_content += bot_config['VerboseMessage'] %\ + substed_content += self._bot_config['VerboseMessage'] %\ (self._var_regex_str % {'var': value, 'cont': u''})
# 2nd stage: conditional metadata substitution (DRTRIGON-132) @@ -318,7 +323,6 @@
Returns a tuple containig the new content with tags substituted and a list of those tags. - """
substed_tags = [] # DRTRIGON-73 @@ -345,17 +349,17 @@ datetime.timedelta(microseconds=1))
pywikibot.output(u'CRON delay for execution: %.3f (<= %i)' - % (delay, bot_config['CRONMaxDelay'])) + % (delay, self._bot_config['CRONMaxDelay']))
- if not (delay <= bot_config['CRONMaxDelay']): + if not (delay <= self._bot_config['CRONMaxDelay']): return (content, substed_tags, metadata)
# 1.) getUrl or wiki text # (security: check url not to point to a local file on the server, # e.g. 'file://' - same as used in xsalt.py) secure = False - for item in [u'http://', u'https://', u'mail://', u'local://', - u'wiki://']: + for item in [u'http://', u'https://', + u'mail://', u'local://', u'wiki://']: secure = secure or (param['url'][:len(item)] == item) param['zip'] = ast.literal_eval(param['zip']) if not secure: @@ -370,8 +374,8 @@ elif (param['url'][:7] == u'mail://'): # DRTRIGON-101 url = param['url'].replace(u'{{@}}', u'@') # e.g. nlwiki mbox = SubsterMailbox( - pywikibot.config.datafilepath(bot_config['data_path'], - bot_config['mbox_file'], '')) + pywikibot.config.datafilepath(self._bot_config['data_path'], + self._bot_config['mbox_file'], '')) external_buffer = mbox.find_data(url) mbox.close() elif (param['url'][:8] == u'local://'): # DRTRIGON-131 @@ -453,7 +457,7 @@ scope.update( locals() ) # (add DATA, *args, ...) scope.update( globals() ) # (add imports and else) if func: - exec(self._code + (bot_config['CodeTemplate'] % func), scope, scope) + exec(self._code + (self._bot_config['CodeTemplate'] % func), scope, scope) external_data = DATA[0] logging.getLogger('subster').debug( external_data )
@@ -541,64 +545,6 @@
return res
-# def data_save(self, outpage, dic, data, comment=None): -# """Stores the content to Wikidata. -# -# @param dic: Original content. -# @type dic: dict -# @param data: New content. -# @type data: dict -# -# Returns nothing, but stores the changed content. -# """ -# # DRTRIGON-130: check for changes and then write/change/set values -# changed = False -# for prop in data: -# pywikibot.output(u'Checking claim with %i values' % len(data[prop])) -# for i, item in enumerate(data[prop]): -# if (i < len(dic[prop])) and \ -# (dic[prop][i][u'mainsnak'][u'datavalue'][u'value'] == item): -# pass # same value; nothing to do -# else: -# # changes; update or create claim -# changed = True -# if (i < len(dic[prop])): -# #print item, dic[prop][i][u'mainsnak'][u'datavalue'][u'value'] -# pywikibot.output(u'Updating claim with value: %s' % item) -# outpage.setclaimvalue(dic[prop][i][u'id'], item, comment=comment) -# else: -# pywikibot.output(u'Creating new claim with value: %s' % item) -# outpage.createclaim(prop, item, comment=comment) -# # search linked items and update them too -# # VERY HACKY, HAS TO BE CONCEPTIONALLY IMPROVED: -# # link any "key = value" pair to any other item by adding "key" -# # to the items 'aliases' (could also use 'description' or even -# # a redirect) -# (key, value) = map(string.strip, item.split('=')) -# for linked in outpage.searchentities(key): -# outpage = pywikibot.DataPage(self.site, linked[u'id']) -# #attr = outpage.getentities() -# attr = linked -# if (u'aliases' in attr) and (key in attr[u'aliases']): -# pywikibot.output(u'Item %s linked to key %s ...' % (outpage.title(asLink=True), key)) -# data = outpage.getentities() -# if u'claims' in data: -# if (data[u'claims'][u'p32'][0][u'mainsnak'][u'datavalue'][u'value'].strip() == value): -# pywikibot.output(u'... ok') -# continue -# changed = True -# pywikibot.output(u'... updating claim with value: %s' % value) -# outpage.setclaimvalue(data[u'claims'][u'p32'][0][u'id'], value, comment=comment) -# else: -# changed = True -# pywikibot.output(u'... creating new claim with value: %s' % value) -# outpage.createclaim(prop, value, comment=comment) -# # speed-up by setting everything at once (in one single write attempt) -# #outpage.editentity(data = {u'claims': data}) -# #outpage.setitem() -# -# return changed - def data_save(self, page, data): """Stores the content to Wikidata.
@@ -611,29 +557,30 @@ """ # DRTRIGON-130: check for changes and then write/change/set values datapage = pywikibot.DataPage(self.site, page.title()) - links = datapage.searchentities(u'%s:%s' % (pywikibot.config.usernames[self.site.family.name][self.site.lang], datapage.title().split(u':')[1])) + links = datapage.searchentities(u'%s:%s' % (self._bot_config['BotName'], datapage.title().split(u':')[1])) for element in links: - item = element[u'aliases'][0].split(u':')[2] + propid = self._bot_config['data_PropertyId'] + el = element[u'aliases'][0].split(u':') + item = el[2] if item not in data: pywikibot.output(u'Value "%s" not found.' % (item,)) continue + if len(el) > 3: + propid = el[3]
dataoutpage = pywikibot.DataPage(self.site, element['id']) - #dataoutpage.createclaim(u'p38', u'{"entity-type":"quantity", "numeric-id":1}') - #dataoutpage = page.toggleTalkPage()
# check for changes and then write/change/set values summary = u'Bot: update data because of configuration on %s.' % page.title(asLink=True) buf = dataoutpage.get() - propid = 217 # just a cheat to start with ... claim = [ claim for claim in buf[u'claims'] if (claim['m'][1] == propid) ] - #if buf.strip().splitlines()[-1].split(u'/')[-1].strip() != data[item]: if (not claim) or (claim[0]['m'][3] != data[item]): pywikibot.output(u'%s in %s <--- %s = %s' %\ (element[u'aliases'][0], dataoutpage.title(asLink=True), item, data[item]))
- #dataoutpage.put(buf + u'\n' + out, comment=summary) - dataoutpage.editclaim(u'p%i' % propid, data[item], comment=summary) + dataoutpage.editclaim(u'p%s' % propid, data[item], +# refs={(self._bot_config['data_PropertyId'], datapage.title()),}, + comment=summary)
def get_var_regex(self, var, cont='.*?'): """Get regex used/needed to find the tags to replace.
Modified: trunk/pywikipedia/subster_irc.py =================================================================== --- trunk/pywikipedia/subster_irc.py 2013-04-16 03:09:20 UTC (rev 11370) +++ trunk/pywikipedia/subster_irc.py 2013-04-16 16:37:10 UTC (rev 11371) @@ -37,11 +37,8 @@ import time import thread
-bot_config = { 'BotName': pywikibot.config.usernames[pywikibot.config.family][pywikibot.config.mylang], +# Configuration imported from 'subster.py'.
- 'ConfCSSconfig': u'User:DrTrigon/DrTrigonBot/subster-config.css', -} - # debug tools # (look at 'bot_control.py' and 'subster.py' for more info) debug = [] @@ -71,7 +68,8 @@ self.do_refresh_References()
# init constants - self._ConfCSSconfigPage = pywikibot.Page(self.site, bot_config['ConfCSSconfig']) + self._BotName = subster.bot_config['BotName'] + self._ConfCSSconfigPage = pywikibot.Page(self.site, subster.bot_config['ConfCSSconfig']) self._difflink = [] if self._ConfCSSconfigPage.exists(): exec(self._ConfCSSconfigPage.get()) # with variable: bot_config_wiki @@ -83,7 +81,7 @@ return # print match.groups(), match.group('page'), match.group('user') user = match.group('user').decode(self.site.encoding()) - if user == bot_config['BotName']: + if user == self._BotName: return #if botlist.isBot(user): # return