jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/371973 ) Change subject: [bugfix, i18n, PEP8] Make reflinks.py work smoothly ...................................................................... [bugfix, i18n, PEP8] Make reflinks.py work smoothly - Allow running without localized stop page - Allow running without localized dead link template - Remove references to DumZiBoT as it is retired - Fix unreachable code to make limit and stop page features work - Fix double brackets when stop page edited - Fix error with requests implementation (T159645, T155509) - Fix Python 3.x and PEP8 errors Bug: T123789 Bug: T159645 Bug: T155509 Change-Id: Iae0d25523fd68ac95b351f2ac9e3cc8484277df4 --- M scripts/reflinks.py 1 file changed, 37 insertions(+), 31 deletions(-) Approvals: jenkins-bot: Verified Xqt: Looks good to me, approved diff --git a/scripts/reflinks.py b/scripts/reflinks.py index c178b0c..1d841b8 100755 --- a/scripts/reflinks.py +++ b/scripts/reflinks.py @@ -12,15 +12,12 @@ The bot checks every 20 edits a special stop page. If the page has been edited, it stops. -DumZiBoT is running that script on en: & fr: at every new dump, running it on -de: is not allowed anymore. +Warning: Running this script on German Wikipedia is not allowed anymore. As it uses it, you need to configure noreferences.py for your wiki, or it will not work. pdfinfo is needed for parsing pdf titles. - -See [[:en:User:DumZiBoT/refLinks]] for more information on the bot. &params; @@ -84,7 +81,7 @@ # should be moved to MediaWiki Pywikibot manual -stopPage = { +stop_page = { 'fr': u'Utilisateur:DumZiBoT/EditezCettePagePourMeStopper', 'da': u'Bruger:DumZiBoT/EditThisPageToStopMe', 'de': u'Benutzer:DumZiBoT/EditThisPageToStopMe', @@ -102,7 +99,6 @@ deadLinkTag = { 'fr': u'[%s] {{lien mort}}', 'da': u'[%s] {{dødt link}}', - 'de': u'', 'fa': u'[%s] {{پیوند مرده}}', 'he': u'{{קישור שבור}}', 'hu': u'[%s] {{halott link}}', @@ -215,8 +211,14 @@ def refDead(self): """Dead link, tag it with a {{dead link}}.""" - tag = i18n.translate(self.site, deadLinkTag) % self.link - return '<ref%s>%s</ref>' % (self.refname, tag) + tag = i18n.translate(self.site, deadLinkTag) + if not tag: + dead_link = self.refLink() + elif '%s' in tag: + dead_link = '<ref%s>%s</ref>' % (self.refname, tag % self.link) + else: + dead_link = '<ref%s>%s</ref>' % (self.refname, tag) + return dead_link def transform(self, ispdf=False): """Normalize the title.""" @@ -408,8 +410,6 @@ self.msg = i18n.twtranslate(self.site, 'reflinks-msg', locals()) else: self.msg = self.getOption('summary') - self.stopPage = pywikibot.Page(self.site, - i18n.translate(self.site, stopPage)) local = i18n.translate(self.site, badtitles) if local: @@ -419,12 +419,15 @@ self.titleBlackList = re.compile(bad, re.I | re.S | re.X) self.norefbot = noreferences.NoReferencesBot(None, verbose=False) self.deduplicator = DuplicateReferences() - try: - self.stopPageRevId = self.stopPage.latest_revision_id - except pywikibot.NoPage: - pywikibot.output(u'The stop page %s does not exist' - % self.stopPage.title(asLink=True)) - raise + + self.site_stop_page = i18n.translate(self.site, stop_page) + if self.site_stop_page: + self.stop_page = pywikibot.Page(self.site, self.site_stop_page) + if self.stop_page.exists(): + self.stop_page_rev_id = self.stop_page.latest_revision_id + else: + pywikibot.warning('The stop page %s does not exist' + % self.stop_page.title(asLink=True)) # Regex to grasp content-type meta HTML tag in HTML source self.META_CONTENT = re.compile(br'(?i)<meta[^>]*content\-type[^>]*>') @@ -454,7 +457,7 @@ """ pywikibot.output(u'PDF file.') fd, infile = tempfile.mkstemp() - urlobj = os.fdopen(fd, 'r+w') + urlobj = os.fdopen(fd, 'w+') urlobj.write(f.content) try: @@ -473,9 +476,9 @@ pywikibot.output(u'pdfinfo value error.') except OSError: pywikibot.output(u'pdfinfo OS error.') - except: # Ignore errors + except Exception: # Ignore errors pywikibot.output(u'PDF processing error.') - pass + pywikibot.exception() finally: urlobj.close() os.unlink(infile) @@ -583,7 +586,7 @@ new_text = new_text.replace(match.group(), repl) continue - linkedpagetext = f.content + linkedpagetext = f.raw except UnicodeError: # example : http://www.adminet.com/jo/20010615¦/ECOC0100037D.html # in [[fr:Cyanure]] @@ -709,12 +712,13 @@ new_text = self.norefbot.addReferences(new_text) new_text = self.deduplicator.process(new_text) + old_text = page.text - self.userPut(page, page.text, new_text, summary=self.msg, + self.userPut(page, old_text, new_text, summary=self.msg, ignore_save_related_errors=True, ignore_server_errors=True) - if new_text == page.text: + if new_text == old_text: continue else: editedpages += 1 @@ -723,15 +727,17 @@ pywikibot.output('Edited %s pages, stopping.' % self.getOption('limit')) return - if editedpages % 20 == 0: - pywikibot.output(color_format( - '{lightgreen}Checking stop page...{default}')) - actualRev = self.stopPage.latest_revision_id - if actualRev != self.stopPageRevId: - pywikibot.output( - u'[[%s]] has been edited : Someone wants us to stop.' - % self.stopPage) - return + if self.site_stop_page and editedpages % 20 == 0: + self.stop_page = pywikibot.Page(self.site, self.site_stop_page) + if self.stop_page.exists(): + pywikibot.output(color_format( + '{lightgreen}Checking stop page...{default}')) + actual_rev = self.stop_page.latest_revision_id + if actual_rev != self.stop_page_rev_id: + pywikibot.output( + '%s has been edited : Someone wants us to stop.' + % self.stop_page.title(asLink=True)) + return def main(*args): -- To view, visit https://gerrit.wikimedia.org/r/371973 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Iae0d25523fd68ac95b351f2ac9e3cc8484277df4 Gerrit-PatchSet: 10 Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-Owner: Dvorapa &lt;dvorapa(a)seznam.cz&gt; Gerrit-Reviewer: Dalba &lt;dalba.wiki(a)gmail.com&gt; Gerrit-Reviewer: Dvorapa &lt;dvorapa(a)seznam.cz&gt; Gerrit-Reviewer: Framawiki &lt;framawiki(a)tools.wmflabs.org&gt; Gerrit-Reviewer: John Vandenberg &lt;jayvdb(a)gmail.com&gt; Gerrit-Reviewer: Magul &lt;tomasz.magulski(a)gmail.com&gt; Gerrit-Reviewer: Matěj Suchánek &lt;matejsuchanek97(a)gmail.com&gt; Gerrit-Reviewer: Mpaa &lt;mpaa.wiki(a)gmail.com&gt; Gerrit-Reviewer: Xqt &lt;info(a)gno.de&gt; Gerrit-Reviewer: Zoranzoki21 &lt;zorandori4444(a)gmail.com&gt; Gerrit-Reviewer: jenkins-bot <>