Pywikipedia-l May 2008

pywikipedia-l@lists.wikimedia.org

32 participants
376 discussions

SVN: [5301] trunk/pywikipedia/reflinks.py
by nicdumz＠svn.wikimedia.org 03 May '08

03 May '08

Revision: 5301 Author: nicdumz Date: 2008-05-03 13:05:23 +0000 (Sat, 03 May 2008) Log Message: ----------- Reminder : Do not commit to fix something you were reported broken when working on some other stuff. Modified Paths: -------------- trunk/pywikipedia/reflinks.py Modified: trunk/pywikipedia/reflinks.py =================================================================== --- trunk/pywikipedia/reflinks.py 2008-05-03 12:48:24 UTC (rev 5300) +++ trunk/pywikipedia/reflinks.py 2008-05-03 13:05:23 UTC (rev 5301) @@ -90,7 +90,8 @@ ur'^\[\]\s<>"]+$[^\[\]\s<>"]+[^\[\]\s\.:;\\,<>\?"]+|'+ # unbracketed without () ur'[^\[\]\s<>"]+[^\[\]\s$\.:;\\,<>\?"]+|[^\[\]\s<>"]+))[!?,\s]*\]?\s*</ref>') -listof404pages = 'http://www.twoevils.org/files/wikipedia/404-links.txt' +#http://www.twoevils.org/files/wikipedia/404-links.txt.gz +listof404pages = '404-links.txt' class XmlDumpPageGenerator: def __init__(self, xmlFilename, xmlStart, namespaces): @@ -286,7 +287,11 @@ Runs the Bot """ wikipedia.setAction(wikipedia.translate(self.site, msg)) - deadLinks = codecs.open(listof404pages, 'r', 'latin_1').read() + try: + deadLinks = codecs.open(listof404pages, 'r', 'latin_1').read() + except IOError: + wikipedia.output('You need to download http://www.twoevils.org/files/wikipedia/404-links.txt.gz and to ungzip it in the same directory') + raise socket.setdefaulttimeout(30) editedpages = 0 for page in self.generator: @@ -322,7 +327,7 @@ headers = f.info() contentType = headers.getheader('Content-Type') if contentType and not self.MIME.search(contentType): - if ref.link.lower().endswith('.pdf') and not ignorepdf: + if ref.link.lower().endswith('.pdf') and not self.ignorepdf: # If file has a PDF suffix self.getPDFTitle(ref, f) else:

1 0

SVN: [5300] trunk/pywikipedia/reflinks.py
by nicdumz＠svn.wikimedia.org 03 May '08

03 May '08

Revision: 5300 Author: nicdumz Date: 2008-05-03 12:48:24 +0000 (Sat, 03 May 2008) Log Message: ----------- * Correcting linklist URL * adding -ingnorepdf Modified Paths: -------------- trunk/pywikipedia/reflinks.py Modified: trunk/pywikipedia/reflinks.py =================================================================== --- trunk/pywikipedia/reflinks.py 2008-05-03 07:59:59 UTC (rev 5299) +++ trunk/pywikipedia/reflinks.py 2008-05-03 12:48:24 UTC (rev 5300) @@ -27,6 +27,9 @@ -xmlstart Page to start with when using an XML dump +-ignorepdf Do not handle PDF files (handy if you use Windows and + can't get pdfinfo) + Basic pagegenerators commands, -page, etc... """ # (C) 2008 - Nicolas Dumazet ( en:User:NicDumZ ) @@ -87,7 +90,7 @@ ur'^\[\]\s<>"]+$[^\[\]\s<>"]+[^\[\]\s\.:;\\,<>\?"]+|'+ # unbracketed without () ur'[^\[\]\s<>"]+[^\[\]\s$\.:;\\,<>\?"]+|[^\[\]\s<>"]+))[!?,\s]*\]?\s*</ref>') -listof404pages = 'http://www.twoevils.org/files/wikipedia/404-links.txt.gz' +listof404pages = 'http://www.twoevils.org/files/wikipedia/404-links.txt' class XmlDumpPageGenerator: def __init__(self, xmlFilename, xmlStart, namespaces): @@ -183,10 +186,11 @@ self.title = self.title.title() class ReferencesRobot: - def __init__(self, generator, acceptall = False, limit = None): + def __init__(self, generator, acceptall = False, limit = None, ignorepdf = False ): self.generator = generator self.acceptall = acceptall self.limit = limit + self.ignorepdf = ignorepdf self.site = wikipedia.getSite() self.stopPage = wikipedia.translate(self.site, stopPage) self.stopPageRevId = wikipedia.Page(self.site, @@ -318,7 +322,7 @@ headers = f.info() contentType = headers.getheader('Content-Type') if contentType and not self.MIME.search(contentType): - if ref.link.lower().endswith('.pdf'): + if ref.link.lower().endswith('.pdf') and not ignorepdf: # If file has a PDF suffix self.getPDFTitle(ref, f) else: @@ -490,6 +494,7 @@ PageTitles = [] xmlFilename = None always = False + ignorepdf = False limit = None namespaces = [] generator = None @@ -505,6 +510,8 @@ wikipedia.setAction(arg[9:]) elif arg == '-always': always = True + elif arg == '-ignorepdf': + ignorepdf= True elif arg.startswith('-limit:'): limit = int(arg[7:]) elif arg.startswith('-xmlstart'): @@ -538,7 +545,7 @@ sys.exit() generator = pagegenerators.PreloadingGenerator(generator, pageNumber = 50) generator = pagegenerators.RedirectFilterPageGenerator(generator) - bot = ReferencesRobot(generator, always, limit) + bot = ReferencesRobot(generator, always, limit, ignorepdf) bot.run() if __name__ == "__main__":

1 0

SVN: [5299] trunk/pywikipedia/reflinks.py
by nicdumz＠svn.wikimedia.org 03 May '08

03 May '08

Revision: 5299 Author: nicdumz Date: 2008-05-03 07:59:59 +0000 (Sat, 03 May 2008) Log Message: ----------- some reflinks.py minor doc Modified Paths: -------------- trunk/pywikipedia/reflinks.py Modified: trunk/pywikipedia/reflinks.py =================================================================== --- trunk/pywikipedia/reflinks.py 2008-05-03 07:57:27 UTC (rev 5298) +++ trunk/pywikipedia/reflinks.py 2008-05-03 07:59:59 UTC (rev 5299) @@ -11,6 +11,10 @@ DumZiBoT is running that script on en: & fr: at every new dump, running it on de: is not allowed anymore. +As it uses it, you need to configure noreferences.py for your wiki, or it will not work. + +pdfinfo is needed for parsing pdf titles. + See [[:en:User:DumZiBoT/refLinks]] for more information on the bot. &params;

1 0

SVN: [5298] trunk/pywikipedia/reflinks.py
by nicdumz＠svn.wikimedia.org 03 May '08

03 May '08

Revision: 5298 Author: nicdumz Date: 2008-05-03 07:57:27 +0000 (Sat, 03 May 2008) Log Message: ----------- Adding my reflink.py to the repository. It is stable now, even if some minor issues are being reported from time to time. Added Paths: ----------- trunk/pywikipedia/reflinks.py Added: trunk/pywikipedia/reflinks.py =================================================================== --- trunk/pywikipedia/reflinks.py (rev 0) +++ trunk/pywikipedia/reflinks.py 2008-05-03 07:57:27 UTC (rev 5298) @@ -0,0 +1,544 @@ +# -*- coding: utf-8 -*- +""" +This bot will search for references which are only made of a link +without title, (i.e. <ref>[http://www.google.fr/]</ref> or +<ref>http://www.google.fr/</ref>) and will fetch the html title from +the link to use it as the title of the wiki link in the reference, i.e. +<ref>[http://www.google.fr/search?q=test test - Google Search]</ref> + +The bot checks every 20 edits its talk page and a special stop page : if +one of these page has been edited, it stops. + +DumZiBoT is running that script on en: & fr: at every new dump, running it on de: is not allowed anymore. + +See [[:en:User:DumZiBoT/refLinks]] for more information on the bot. + +&params; + +-limit:n Stops after n edits + +-xml:dump.xml Should be used instead of a simple page fetching + method from pagegenerators.py for performance and + load issues + +-xmlstart Page to start with when using an XML dump + +Basic pagegenerators commands, -page, etc... +""" +# (C) 2008 - Nicolas Dumazet ( en:User:NicDumZ ) +# +# Distributed under the terms of the GPL + +from BeautifulSoup import UnicodeDammit +import sys, re, urllib2, httplib, socket, codecs, ftplib +import wikipedia, pagegenerators, noreferences +import subprocess, tempfile, os + +stopPage = {'fr':u'Utilisateur:DumZiBoT/EditezCettePagePourMeStopper', + 'de':u'Benutzer:DumZiBoT/EditThisPageToStopMe', + 'it':u'Utente:DumZiBoT/EditThisPageToStopMe', + 'en':u'User:DumZiBoT/EditThisPageToStopMe'} + +msg = { 'fr':u'Bot: Correction des refs. mal formatées (cf. [[Utilisateur:DumZiBoT/liensRefs|explications]])', + 'de':u'Bot: Korrektes Referenzformat (siehe [[:en:User:DumZiBoT/refLinks]])', + 'en':u'Bot: Converting bare references, see [[User:DumZiBoT/refLinks|FAQ]]'} + +deadLinkTag = {'fr':u'{{lien mort}}', + 'de':u'', + 'en':u'{{dead link}}'} + +comment = {'fr':u'Titre généré automatiquement', + 'de':u'Automatisch generierter titel', + 'en':u'Bot generated title'} + +soft404 = re.compile(ur'\D404(\D|\Z)|error|errdoc|Not.{0,3}Found|sitedown|eventlog', re.IGNORECASE) +dirIndex = re.compile(ur'^\w+://[^/]+/((default|index)\.(asp|aspx|cgi|htm|html|phtml|mpx|mspx|php|shtml|var))?$', re.IGNORECASE) +domain = re.compile(ur'^(\w+)://(?:www.|)([^/]+)') +badtitles = {'en': + # is + ur'(?is) (test|' + # starts with + +'^\W*(register|registration|(sign|log)[ \-]?in|subscribe|sign[ \-]?up|log[ \-]?on|untitled *(document|page|$))' + # anywhere + +'|(404|page|file).*not( *be)? *found' + # ends with + +'|(register|registration|(sign|log)[ \-]?in|subscribe|sign[ \-]?up|log[ \-]?on)\W*$' + +')', + 'fr': + #is + ur'(?is) (test|' + # starts with + ur'^\W*(register|registration|(sign|log)[ \-]?in|subscribe|sign[ \-]?up|log[ \-]?on|untitled *(document|page|$))' + # anywhere + +'|(404|page|file|site).*(not *found|en +travaux)' + # ends with + +'|(register|registration|(sign|log)[ \-]?in|subscribe|sign[ \-]?up|log[ \-]?on)\W*$' + +')'} + + +linksInRef = re.compile( + # bracketed URLs + ur'(?i)<ref(?P<name>[^>]*)>\s*\[?(?P<url>(?:http|https|ftp)://(?:' + + # unbracketed with() + ur'^\[\]\s<>"]+$[^\[\]\s<>"]+[^\[\]\s\.:;\\,<>\?"]+|'+ + # unbracketed without () + ur'[^\[\]\s<>"]+[^\[\]\s$\.:;\\,<>\?"]+|[^\[\]\s<>"]+))[!?,\s]*\]?\s*</ref>') +listof404pages = 'http://www.twoevils.org/files/wikipedia/404-links.txt.gz' + +class XmlDumpPageGenerator: + def __init__(self, xmlFilename, xmlStart, namespaces): + self.xmlFilename = xmlFilename + self.xmlStart = xmlStart + self.namespaces = namespaces + self.skipping = bool(xmlStart) + self.site = wikipedia.getSite() + + import xmlreader + dump = xmlreader.XmlDump(self.xmlFilename) + self.parser = dump.parse() + + def __iter__(self): + return self + + def next(self): + while True: + try: + entry = self.parser.next() + except StopIteration: + raise + if self.skipping: + if entry.title != self.xmlStart: + continue + self.skipping = False + page=wikipedia.Page(self.site, entry.title) + if not self.namespaces == []: + if page.namespace() not in self.namespaces: + continue + if linksInRef.search(entry.text): + return page + +class RefLink: + def __init__(self, link, name): + self.refname = name + self.link = link + self.site = wikipedia.getSite() + self.linkComment = wikipedia.translate(self.site, comment) + self.url = re.sub(u'#.*', '', self.link) + self.title = None + + def refTitle(self): + return '<ref%s>[%s %s]</ref>' % (self.refname, self.link, self.title, self.linkComment) + + def refLink(self): + return '<ref%s>%s</ref>' % (self.refname, self.link) + + def refDead(self): + tag = wikipedia.translate(self.site, deadLinkTag) + return '<ref%s>[%s]%s</ref>' % (self.refname, self.link, tag) + + def transform(self, ispdf = False): + #convert html entities + if not ispdf: + self.title = wikipedia.html2unicode(self.title) + self.title = re.sub(r'-+', '-', self.title) + #remove formatting, i.e long useless strings + self.title = re.sub(r'[\.+\-=]{4,}', ' ', self.title) + #remove \n and \r and Unicode spaces from titles + self.title = re.sub(r'(?u)\s', ' ', self.title) + self.title = re.sub(r'[\n\r\t]', ' ', self.title) + #remove extra whitespaces + #remove leading and trailing ./;/,/-/_/+/ / + self.title = re.sub(r' +', ' ', self.title.strip(r'=.;,-+_ ')) + + self.avoid_uppercase() + #avoid closing the link before the end + self.title = self.title.replace(']', ']') + #avoid multiple } being interpreted as a template inclusion + self.title = self.title.replace('}}', '}}') + #prevent multiple quotes being interpreted as '' or ''' + self.title = self.title.replace('\'\'', '\''') + self.title = wikipedia.unicode2html(self.title, self.site.encoding()) + + def avoid_uppercase(self): + """ + If title has more than 6 characters and has 60% of uppercase + characters, capitalize() it + """ + if len(self.title) <= 6: + return + nb_upper = 0 + nb_letter = 0 + for letter in self.title: + if letter.isupper(): + nb_upper += 1 + if letter.isalpha(): + nb_letter += 1 + if letter.isdigit(): + return + if float(nb_upper)/(nb_letter+1) > .70: + self.title = self.title.title() + +class ReferencesRobot: + def __init__(self, generator, acceptall = False, limit = None): + self.generator = generator + self.acceptall = acceptall + self.limit = limit + self.site = wikipedia.getSite() + self.stopPage = wikipedia.translate(self.site, stopPage) + self.stopPageRevId = wikipedia.Page(self.site, + self.stopPage).latestRevision() + self.META_CONTENT = re.compile(ur'(?i)<meta[^>]*content\-type[^>]*>') + self.CHARSET = re.compile(ur'(?i)charset\s*=\s*(?P<enc>[^\'";>/]*)') + self.TITLE = re.compile(ur'(?is)(?<=<title>).*?(?=</title>)') + self.NON_HTML = re.compile(ur'(?is)<script[^>]*>.*?</script>|<style[^>]*>.*?</style>||<!\[CDATA\[.*?\]\]>') + str = ur'application/(?:xhtml\+xml|xml)|text/(?:ht|x)ml' + self.MIME = re.compile(str) + self.titleBlackList = re.compile(wikipedia.translate(self.site, badtitles)) + self.norefbot = noreferences.NoReferencesBot(None) + + def put_page(self, page, new): + """ + Prints diffs between orginal and new (text), puts new text for page + """ + wikipedia.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" + % page.title()) + wikipedia.showDiff(page.get(), new) + if not self.acceptall: + choice = wikipedia.inputChoice(u'Do you want to accept ' + + u'these changes?', + ['Yes', 'No', 'All'], + ['y', 'N', 'a'], 'N') + if choice in ['a', 'A']: + self.acceptall = True + if choice in ['y', 'Y']: + page.put_async(new) + if self.acceptall: + try: + page.put(new) + except wikipedia.EditConflict: + wikipedia.output(u'Skipping %s because of edit conflict' + % (page.title(),)) + except wikipedia.SpamfilterError, e: + wikipedia.output(u'Cannot change %s because of blacklist entry %s' % (page.title(), e.url)) + except wikipedia.PageNotSaved, error: + wikipedia.output(u'Error putting page: %s' % (error.args,)) + except wikipedia.LockedPage: + wikipedia.output(u'Skipping %s (locked page)' + % (page.title(),)) + except wikipedia.ServerError, e: + wikipedia.output(u'Server Error : %s' % e) + + def httpError(self, err_num, link, pagetitleaslink): + """Log HTTP Error""" + wikipedia.output(u'HTTP error (%s) for %s on %s' + % (err_num, link, pagetitleaslink), + toStdout = True) + f = codecs.open( + wikipedia.config.datafilepath( + 'reflinks-httpErrorLog', + 'reflinks-%s-%s.txt' % (self.site.family.name, + self.site.lang)), + 'a', 'utf-8') + f.write(u'%s: %s from %s\n' % (err_num, link, pagetitleaslink)) + f.close() + + def getPDFTitle(self, ref, f): + wikipedia.output( u'PDF file.' ) + fd, infile = tempfile.mkstemp() + urlobj = os.fdopen(fd, 'r+w') + urlobj.write(f.read()) + try: + pdfinfo_out = subprocess.Popen([r"pdfinfo","/dev/stdin"], stdin=urlobj, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=False).communicate()[0] + for aline in pdfinfo_out.splitlines(): + if aline.lower().startswith('title'): + ref.title = aline.split(None)[1:] + ref.title = ' '.join(ref.title) + if ref.title != '': wikipedia.output(u'title: ' +ref.title ) + wikipedia.output( u'PDF done.' ) + except ValueError: + wikipedia.output( u'pdfinfo value error.' ) + except OSError: + wikipedia.output( u'pdfinfo OS error.' ) + except: # Ignore errors + wikipedia.output( u'PDF processing error.' ) + pass + finally: + urlobj.close() + os.unlink(infile) + + def replace(self, sub, text, repl): + new_text = text + for match in re.finditer(re.escape(sub), text): + if wikipedia.isDisabled(text, match.start()): + continue + #blablah + + def run(self): + """ + Runs the Bot + """ + wikipedia.setAction(wikipedia.translate(self.site, msg)) + deadLinks = codecs.open(listof404pages, 'r', 'latin_1').read() + socket.setdefaulttimeout(30) + editedpages = 0 + for page in self.generator: + try: + # Load the page's text from the wiki + new_text = page.get() + if not page.canBeEdited(): + wikipedia.output(u"You can't edit page %s" + % page.aslink()) + continue + except wikipedia.NoPage: + wikipedia.output(u'Page %s not found' % page.aslink()) + continue + except wikipedia.IsRedirectPage: + wikipedia.output(u'Page %s is a redirect' % page.aslink()) + continue + + for match in linksInRef.finditer(wikipedia.removeDisabledParts(page.get())): + #for each link to change + link = match.group(u'url') + #debugging purpose + #print link + if u'jstor.org' in link: + #TODO: Clean URL blacklist + continue + + ref = RefLink(link, match.group('name')) + f = None + try: + socket.setdefaulttimeout(20) + f = urllib2.urlopen(ref.url) + #Try to get Content-Type from server + headers = f.info() + contentType = headers.getheader('Content-Type') + if contentType and not self.MIME.search(contentType): + if ref.link.lower().endswith('.pdf'): + # If file has a PDF suffix + self.getPDFTitle(ref, f) + else: + wikipedia.output(u'\03{lightyellow}WARNING\03{default} : media : %s ' % ref.link) + if ref.title: + if not re.match('(?i) *microsoft (word|excel|visio)', ref.title): + ref.transform(ispdf=True) + repl = ref.refTitle() + else: + wikipedia.output('\03{lightyellow}WARNING\03{default} : PDF title blacklisted : %s ' % ref.title) + repl = ref.refLink() + else: + repl = ref.refLink() + new_text = new_text.replace(match.group(), repl) + continue + # Test if the redirect was valid + redir = f.geturl() + if redir != ref.link and domain.findall(redir) == domain.findall(link): + if soft404.search(redir) and not soft404.search(ref.link): + wikipedia.output(u'\03{lightyellow}WARNING\03{default} : Redirect 404 : %s ' % ref.link) + continue + if dirIndex.match(redir) and not dirIndex.match(ref.link): + wikipedia.output(u'\03{lightyellow}WARNING\03{default} : Redirect to root : %s ' % ref.link) + continue + + # Read the first 1,000,000 bytes (0.95 MB) + linkedpagetext = f.read(1000000) + socket.setdefaulttimeout(None) + + except UnicodeError: + #example : http://www.adminet.com/jo/20010615¦/ECOC0100037D.html in [[fr:Cyanure]] + wikipedia.output(u'\03{lightred}Bad link\03{default} : %s in %s' % (ref.url, page.aslink())) + continue + except urllib2.HTTPError, e: + self.httpError(e.code, ref.url, page.aslink()) + if e.code == 410: # 410 Gone, indicates that the resource has been purposely removed + repl = ref.refDead() + new_text = new_text.replace(match.group(), repl) + elif e.code == 404 and (u'\t%s\t' % ref.url in deadLinks): + repl = ref.refDead() + new_text = new_text.replace(match.group(), repl) + continue + except (urllib2.URLError, + socket.error, + IOError, + httplib.error), e: + #except (urllib2.URLError, socket.timeout, ftplib.error, httplib.error, socket.error), e: + wikipedia.output(u'Can\'t get page %s : %s' % (ref.url, e)) + continue + except ValueError: + #Known bug of httplib, google for : + #"httplib raises ValueError reading chunked content" + continue + finally: + if f: + f.close() + + #remove <script>/<style>/comments/CDATA tags + linkedpagetext = self.NON_HTML.sub('', linkedpagetext) + + meta_content = self.META_CONTENT.search(linkedpagetext) + enc = [] + if meta_content: + tag = meta_content.group() + if not contentType: + contentType = tag + s = self.CHARSET.search(tag) + if s: + tmp = s.group('enc').strip("\"' ").lower() + enc.append(tmp) + if tmp in ("gb 2312", "gb2312", "gb-2312", "gb_2312"): + enc.append("gbk") + if tmp in ("shift jis", "shiftjis", "shift-jis", "shift_jis"): + enc.append("shift jis 2004") + enc.append("cp932") + if tmp in ("x euc jp", "x-euc-jp"): + enc.append("euc-jp") + if not contentType: + wikipedia.output(u'No content-type found for %s' % ref.link) + continue + elif not self.MIME.search(contentType): + wikipedia.output(u'\03{lightyellow}WARNING\03{default} : media : %s ' % ref.link) + repl = ref.refLink() + new_text = new_text.replace(match.group(), repl) + continue + + if u'.ru' in ref.link or u'.su' in ref.link: + # see http://www.sci.aha.ru/ATL/ra13a.htm : no server encoding, no page encoding + enc = enc + ['koi8-r', 'windows-1251'] + elif u'.jp' in ref.link: + enc.append("shift jis 2004") + enc.append("cp932") + elif u'.zh' in ref.link: + enc.append("gbk") + + #print(enc) + u = UnicodeDammit(linkedpagetext, overrideEncodings = enc) + #print(u.triedEncodings) + + + if not u.unicode: + #Some page have utf-8 AND windows-1252 characters, + #Can't easily parse them. (~1 on 1000) + repl = ref.refLink() + new_text = new_text.replace(match.group(), repl) + wikipedia.output('%s : Hybrid encoding...' % ref.link) + continue + + + for m in self.TITLE.finditer(u.unicode): + t = m.group() + if t: + ref.title = t + ref.transform() + if ref.title: + break; + + if not ref.title: + repl = ref.refLink() + new_text = new_text.replace(match.group(), repl) + wikipedia.output(u'%s : No title found...' % ref.link) + continue + if enc and u.originalEncoding not in enc: + wikipedia.output(u'\03{lightpurple}ENCODING\03{default} : %s (%s)' % (ref.link, ref.title)) + + if u'Ã©' in ref.title: + repl = ref.refLink() + new_text = new_text.replace(match.group(), repl) + wikipedia.output(u'%s : Hybrid encoding...' % ref.link) + continue + + if self.titleBlackList.search(ref.title): + repl = ref.refLink() + new_text = new_text.replace(match.group(), repl) + wikipedia.output(u'\03{lightred}WARNING\03{default} %s : Blacklisted title (%s)' % (ref.link, ref.title)) + continue + if len(ref.title) > 250: + ref.title = ref.title[:250] + "..." + + repl = ref.refTitle() + new_text = new_text.replace(match.group(), repl) + + if new_text == page.get(): + wikipedia.output('No changes were necessary in %s' + % page.aslink()) + continue + + if self.norefbot.lacksReferences(new_text, verbose=False): + new_text = self.norefbot.addReferences(new_text) + editedpages += 1 + self.put_page(page, new_text) + if self.limit and editedpages >= self.limit: + wikipedia.output('Edited %s pages, stopping.' % self.limit) + return + if editedpages % 20 == 0: + wikipedia.output('\03{lightgreen}Checking stop page...\03{default}') + actualRev = wikipedia.Page(self.site, + self.stopPage).latestRevision() + if actualRev != self.stopPageRevId: + wikipedia.output(u'[[%s]] has been edited : Someone wants us to stop.' % self.stopPage) + return + if self.site.messages(): + wikipedia.output(u'Bot has new messages. Better stop to check.') + return + +def main(): + genFactory = pagegenerators.GeneratorFactory() + + PageTitles = [] + xmlFilename = None + always = False + limit = None + namespaces = [] + generator = None + for arg in wikipedia.handleArgs(): + if arg.startswith('-page:'): + PageTitles.append(arg[6:]) + elif arg.startswith('-namespace:'): + try: + namespaces.append(int(arg[11:])) + except ValueError: + namespaces.append(arg[11:]) + elif arg.startswith('-summary:'): + wikipedia.setAction(arg[9:]) + elif arg == '-always': + always = True + elif arg.startswith('-limit:'): + limit = int(arg[7:]) + elif arg.startswith('-xmlstart'): + if len(arg) == 9: + xmlStart = wikipedia.input( + u'Please enter the dumped article to start with:') + else: + xmlStart = arg[10:] + elif arg.startswith('-xml'): + if len(arg) == 4: + xmlFilename = wikipedia.input( + u'Please enter the XML dump\'s filename:') + else: + xmlFilename = arg[5:] + else: + generator = genFactory.handleArg(arg) + + if xmlFilename: + try: + xmlStart + except NameError: + xmlStart = None + generator = XmlDumpPageGenerator(xmlFilename, xmlStart, namespaces) + elif PageTitles: + pages= [wikipedia.Page(wikipedia.getSite(), PageTitle) for PageTitle in PageTitles] + generator = iter(pages) + if not generator: + # syntax error, show help text from the top of this file + wikipedia.showHelp('reflinks') + wikipedia.stopme() + sys.exit() + generator = pagegenerators.PreloadingGenerator(generator, pageNumber = 50) + generator = pagegenerators.RedirectFilterPageGenerator(generator) + bot = ReferencesRobot(generator, always, limit) + bot.run() + +if __name__ == "__main__": + try: + main() + finally: + wikipedia.stopme() Property changes on: trunk/pywikipedia/reflinks.py ___________________________________________________________________ Name: svn:eol-style + native

1 0

[ pywikipediabot-Bugs-1797503 ] category.py
by SourceForge.net 03 May '08

03 May '08

Bugs item #1797503, was opened at 2007-09-18 15:05 Message generated for change (Comment added) made by sf-robot You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1797503&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None >Status: Closed Resolution: None Priority: 5 Private: No Submitted By: DarkoNeko (darkoneko) Assigned to: Nobody/Anonymous (nobody) Summary: category.py Initial Comment: evt : cmd.exe on windows XP family edition command used : C:\Program Files\TortoiseSVN\pywikipedia>python category.py move -from:"Cumuni di Sicilia" -to:"Cumuna di Sicilia" -lang:co error message : There are more articles in Category:Cumuni di Sicilia. Getting [[Category:Cumuni di Sicilia]] starting at Mistirjancu" class="new... Changing page [[co:Longi]] WARNING: No character set found. Category page detection is not bug free. Please report this error! substring not found Changing page [[co:Lucca Sicula]] (a few other pages) Changing page [[co:Marineu]] Getting [[Category:Cumuni di Sicilia]]... There are more articles in Category:Cumuni di Sicilia. Getting [[Category:Cumuni di Sicilia]] starting at Vita+%28Sicilia%29" class="ne w... WARNING: No character set found. Category page detection is not bug free. Please report this error! Dumping to category.dump.bz2, please wait... Traceback (most recent call last): File "category.py", line 832, in <module> bot.run() File "category.py", line 365, in run subcategories = self.oldCat.subcategoriesList(recurse = False) File "C:\Program Files\TortoiseSVN\pywikipedia\catlib.py", line 298, in subcat egoriesList for cat in self.subcategories(recurse): File "C:\Program Files\TortoiseSVN\pywikipedia\catlib.py", line 284, in subcat egories for tag, subcat in self._getContentsAndSupercats(recurse): File "C:\Program Files\TortoiseSVN\pywikipedia\catlib.py", line 124, in _getCo ntentsAndSupercats for tag, page in self._parseCategory(purge, startFrom): File "C:\Program Files\TortoiseSVN\pywikipedia\catlib.py", line 204, in _parse Category ibegin = txt.index('') # does not work for cats withou t text ValueError: substring not found apparently an error when recovering the page list. ---------------------------------------------------------------------- >Comment By: SourceForge Robot (sf-robot) Date: 2008-05-02 19:20 Message: Logged In: YES user_id=1312539 Originator: NO This Tracker item was closed automatically by the system. It was previously set to a Pending status, and the original submitter did not respond within 14 days (the time period specified by the administrator of this Tracker). ---------------------------------------------------------------------- Comment By: Russell Blau (russblau) Date: 2008-04-18 08:41 Message: Logged In: YES user_id=855050 Originator: NO If this error is still occurring, please provide updated information. If no information is received, this bug will be closed automatically. ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1797503&group_…

1 0

[ pywikipediabot-Bugs-1803615 ] Another bug in category.py
by SourceForge.net 03 May '08

03 May '08

Bugs item #1803615, was opened at 2007-09-27 07:10 Message generated for change (Comment added) made by sf-robot You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1803615&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None >Status: Closed Resolution: None Priority: 5 Private: No Submitted By: Nobody/Anonymous (nobody) Assigned to: Nobody/Anonymous (nobody) Summary: Another bug in category.py Initial Comment: I have no idea how this bug occurs, but here it is. It doesn't recognize that a simple category already exists in this case, which is strange, since the category I'm adding is Landskrona BoIS and in this case the DEFAULTSORT template is NOT being used in the article. Current categories: * Kategori:Födda 1984 * Kategori:Spelare i Häljarps IF * Kategori:Spelare i IFK Hässleholm <b>* Kategori:Spelare i Landskrona BoIS</b> * Kategori:Svenska fotbollsspelare Adding [[Kategori:Spelare i Landskrona BoIS|Dahlgren, Mikael]] Very peculiar. ---------------------------------------------------------------------- >Comment By: SourceForge Robot (sf-robot) Date: 2008-05-02 19:20 Message: Logged In: YES user_id=1312539 Originator: NO This Tracker item was closed automatically by the system. It was previously set to a Pending status, and the original submitter did not respond within 14 days (the time period specified by the administrator of this Tracker). ---------------------------------------------------------------------- Comment By: Russell Blau (russblau) Date: 2008-04-18 08:39 Message: Logged In: YES user_id=855050 Originator: NO If this bug is still occurring, please provide updated information including the specific command line used to run the bot. If no information is provided, this bug will be closed automatically. ---------------------------------------------------------------------- Comment By: Nobody/Anonymous (nobody) Date: 2007-09-27 07:10 Message: Logged In: NO I didn't know that HTML code was disabled in bug descriptions... The HTML code is of course not being printed in output. ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1803615&group_…

1 0

SVN: [5297] branches/rewrite/pywikibot
by russblau＠svn.wikimedia.org 03 May '08

03 May '08

Revision: 5297 Author: russblau Date: 2008-05-02 20:29:49 +0000 (Fri, 02 May 2008) Log Message: ----------- page.py: to normalize titles, use pywikibot.Page(pywikibot.Link("foo:bar")) in place of Page(mysite, "foo:bar"). site.py: fix bug in namespace loading. comms/threadedhttp.py: more logging. Modified Paths: -------------- branches/rewrite/pywikibot/comms/threadedhttp.py branches/rewrite/pywikibot/page.py branches/rewrite/pywikibot/site.py Modified: branches/rewrite/pywikibot/comms/threadedhttp.py =================================================================== --- branches/rewrite/pywikibot/comms/threadedhttp.py 2008-05-02 14:32:04 UTC (rev 5296) +++ branches/rewrite/pywikibot/comms/threadedhttp.py 2008-05-02 20:29:49 UTC (rev 5297) @@ -34,7 +34,8 @@ try: import pkg_resources except ImportError: - print "Error : You need the python module setuptools to use this module" + logging.critical( + "Error : You need the python module setuptools to use this module") sys.exit(1) pkg_resources.require("httplib2") import httplib2 @@ -194,7 +195,8 @@ follow_redirects = self.follow_redirects self.follow_redirects = False logging.debug('%r' % ( - (uri, method, body, headers, max_redirects, connection_type),)) + (uri.replace("%7C","|"), + method, body, headers, max_redirects, connection_type),)) try: (response, content) = httplib2.Http.request( self, uri, method, body, headers, Modified: branches/rewrite/pywikibot/page.py =================================================================== --- branches/rewrite/pywikibot/page.py 2008-05-02 14:32:04 UTC (rev 5296) +++ branches/rewrite/pywikibot/page.py 2008-05-02 20:29:49 UTC (rev 5297) @@ -75,7 +75,6 @@ "Invalid namespace '%i' for site %s." % (ns, source.sitename())) self._ns = ns - title = title[0].upper() + title[1:] if ns and not title.startswith(source.namespace(ns)+u":"): title = source.namespace(ns) + u":" + title elif not ns and u":" in title: @@ -83,7 +82,6 @@ nsindex = source.getNamespaceIndex(title[ :pos]) if nsindex: self._ns = nsindex - title = title[:pos+1] + title[pos+1].upper() + title[pos+2:] if u"#" in title: title, self._section = title.split(u"#", 1) else: @@ -1381,10 +1379,10 @@ % text) t = t[t.index(u":"): ].lstrip(u":").lstrip(u" ") if prefix in fam.langs.keys(): - newsite = pywikibot.Site(pre, fam) + newsite = pywikibot.Site(prefix, fam) else: - otherlang = self.site.lang - familyName = fam.get_known_families(site=self.site)[pre] + otherlang = self.site.language() + familyName = fam.get_known_families(site=self.site)[prefix] if familyName in ['commons', 'meta']: otherlang = familyName try: Modified: branches/rewrite/pywikibot/site.py =================================================================== --- branches/rewrite/pywikibot/site.py 2008-05-02 14:32:04 UTC (rev 5296) +++ branches/rewrite/pywikibot/site.py 2008-05-02 20:29:49 UTC (rev 5297) @@ -168,9 +168,9 @@ def getNamespaceIndex(self, namespace): """Given a namespace name, return its int index, or None if invalid.""" - for ns in self._namespaces: + for ns in self.namespaces(): if namespace.lower() in [name.lower() - for name in self._namespaces[ns]]: + for name in self.namespaces()[ns]]: return ns return None @@ -490,10 +490,9 @@ values for this namespace. """ - self.getsiteinfo() if all: - return self._namespaces[num] - return self._namespaces[num][0] + return self.namespaces()[num] + return self.namespaces()[num][0] def getpageinfo(self, page): """Load page info from api and save in page attributes"""

2 1

SVN: [5296] trunk/pywikipedia/checkimages.py
by filnik＠svn.wikimedia.org 02 May '08

02 May '08

Revision: 5296 Author: filnik Date: 2008-05-02 14:32:04 +0000 (Fri, 02 May 2008) Log Message: ----------- Improving the duplicates detection and report Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2008-05-02 12:47:15 UTC (rev 5295) +++ trunk/pywikipedia/checkimages.py 2008-05-02 14:32:04 UTC (rev 5296) @@ -316,7 +316,18 @@ 'it': u"{{subst:Utente:Filbot/Template_insufficiente|%s}} --~~~~", 'ta': None, } - +# Stub - will make it better in future (no time now) +duplicatesText = { + 'commons':u'\n{{Dupe|__image__}}', + 'en':None, + 'it':u'\n{{Cancella subito|Immagine doppia di __image__}}', + } +duplicatesRegex = { + 'commons':r'\{\{(?:[Tt]emplate:|)[Dd]upe[|}]', + 'en':None, + 'it':r'\{\{(?:[Tt]emplate:|)[Cc]ancella[ _]subito[|}]', + } + # Add your project (in alphabetical order) if you want that the bot start project_inserted = [u'ar', u'commons', u'de', u'en', u'ja', u'hu', u'it', u'ta', u'zh'] @@ -569,6 +580,10 @@ def checkImageDuplicated(self, image): """ Function to check the duplicated images. """ + # {{Dupe|Image:Blanche_Montel.jpg}} + # report(unvertext, imageName, notification, head) + dupText = wikipedia.translate(self.site, duplicatesText) + dupRegex = wikipedia.translate(self.site, duplicatesRegex) self.image = image duplicateRegex = r'\n\*(?:\[\[:Image:%s\]\] has the following duplicates:|\*\[\[:Image:%s\]\])$' % (self.convert_to_url(self.image), self.convert_to_url(self.image)) imagePage = wikipedia.ImagePage(self.site, 'Image:%s' % self.image) @@ -594,8 +609,20 @@ for duplicate in duplicates: if self.convert_to_url(duplicate) == self.convert_to_url(self.image): continue # the image itself, not report also this as duplicate - repme += "\n**[[:Image:%s]]" % self.convert_to_url(duplicate) - self.report_image(self.image, self.rep_page, self.com, repme, addings = False, regex = duplicateRegex) + repme += "\n**[[:Image:%s]]" % self.convert_to_url(duplicate) + result = self.report_image(self.image, self.rep_page, self.com, repme, addings = False, regex = duplicateRegex) + if result and not dupText == None and not dupRegex == None: + for duplicate in duplicates: + if self.convert_to_url(duplicate) == self.convert_to_url(self.image): + continue # the image itself, not report also this as duplicate + DupePage = wikipedia.Page(self.site, u'Image:%s' % duplicate) + try: + DupPageText = DupePage.get() + except wikipedia.NoPage: + continue # The page doesn't exists + if re.findall(dupRegex, DupPageText) == []: + wikipedia.output(u'Adding the duplicate template in the image...') + report(re.sub(r'__image__', r'%s' % self.image, dupText), duplicate) return True # Ok - No problem. Let's continue the checking phase def report_image(self, image, rep_page = None, com = None, rep_text = None, addings = True, regex = None): @@ -690,13 +717,14 @@ # I've seen that the report class before (the main) was to long to be called so, # here there is a function that has all the settings, so i can call it once ^__^ -def report(newtext, image, notification, head, notification2 = None, unver = True, commx = None, bot_list = bot_list): +def report(newtext, image, notification = None, head = None, notification2 = None, unver = True, commx = None, bot_list = bot_list): # Adding the bot's nickname at the notification text if needed. botolist = wikipedia.translate(wikipedia.getSite(), bot_list) project = wikipedia.getSite().family.name bot = config.usernames[project] botnick = bot[wikipedia.getSite().lang] - notification = re.sub('__botnick__', botnick, notification) + if notification != None: + notification = re.sub('__botnick__', botnick, notification) if notification2 != None: notification2 = re.sub('__botnick__', botnick, notification2) # Ok, done, let's loop. @@ -727,14 +755,17 @@ else: if resPutMex == False: break - try: - run.put_talk(notification, head, notification2, commx) - except wikipedia.EditConflict: - wikipedia.output(u"Edit Conflict! Retrying...") + if notification != None and head != None: try: run.put_talk(notification, head, notification2, commx) - except: - wikipedia.output(u"Another error... skipping the user..") + except wikipedia.EditConflict: + wikipedia.output(u"Edit Conflict! Retrying...") + try: + run.put_talk(notification, head, notification2, commx) + except: + wikipedia.output(u"Another error... skipping the user..") + break + else: break else: break

1 0

[ pywikipediabot-Feature Requests-1956119 ] intwerwiki to non existing anchor
by SourceForge.net 02 May '08

02 May '08

Feature Requests item #1956119, was opened at 2008-05-02 05:52 Message generated for change (Tracker Item Submitted) made by Item Submitter You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603141&aid=1956119&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None Status: Open Priority: 5 Private: No Submitted By: Nobody/Anonymous (nobody) Assigned to: Nobody/Anonymous (nobody) Summary: intwerwiki to non existing anchor Initial Comment: Please, add possibility to give interwiki link to anchor whis is not one of headers of article. Example: There are articles Microsoft .NET and .NET Framework in several languages, but in en: is only .NET Framework. But some people repeatly add link to .NET Framework to the second group of articles. If there is link [[en:.NET_Framework#top]], it works correctly for browser, but bot wants to remove this link ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603141&aid=1956119&group_…

1 0

SVN: [5295] trunk/pywikipedia/checkimages.py
by filnik＠svn.wikimedia.org 02 May '08

02 May '08

Revision: 5295 Author: filnik Date: 2008-05-02 12:47:15 +0000 (Fri, 02 May 2008) Log Message: ----------- Fixing a bit the duplicate function Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2008-05-02 11:05:19 UTC (rev 5294) +++ trunk/pywikipedia/checkimages.py 2008-05-02 12:47:15 UTC (rev 5295) @@ -570,7 +570,7 @@ def checkImageDuplicated(self, image): """ Function to check the duplicated images. """ self.image = image - duplicateRegex = r'\n\*(?:\[\[:Image:%s\]\] has the following duplicates:|\*\[\[:Image:%s\]\])$' % (self.image, self.image) + duplicateRegex = r'\n\*(?:\[\[:Image:%s\]\] has the following duplicates:|\*\[\[:Image:%s\]\])$' % (self.convert_to_url(self.image), self.convert_to_url(self.image)) imagePage = wikipedia.ImagePage(self.site, 'Image:%s' % self.image) wikipedia.output(u'Checking if %s has duplicates...' % image) get_hash = self.site.getUrl('/w/api.php?action=query&format=xml&titles=Image:%s&prop=imageinfo&iiprop=sha1' % self.convert_to_url(self.image)) @@ -590,11 +590,11 @@ wikipedia.output(u'%s has a duplicate! Reporting it...' % self.image) else: wikipedia.output(u'%s has %s duplicates! Reporting them...' % (self.image, len(duplicates) - 1)) - repme = "\n*[[:Image:%s]] has the following duplicates:" % self.image + repme = "\n*[[:Image:%s]] has the following duplicates:" % self.convert_to_url(self.image) for duplicate in duplicates: - if duplicate == self.image: + if self.convert_to_url(duplicate) == self.convert_to_url(self.image): continue # the image itself, not report also this as duplicate - repme += "\n**[[:Image:%s]]" % duplicate + repme += "\n**[[:Image:%s]]" % self.convert_to_url(duplicate) self.report_image(self.image, self.rep_page, self.com, repme, addings = False, regex = duplicateRegex) return True # Ok - No problem. Let's continue the checking phase

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

Pywikipedia-l May 2008