pywikibot

pywikibot@lists.wikimedia.org

6828 discussions

[Pywikipedia-l] SVN: [3937] trunk/pywikipedia/wikipedia.py

by valhallasw＠svn.wikimedia.org

Revision: 3937 Author: valhallasw Date: 2007-08-01 21:53:08 +0000 (Wed, 01 Aug 2007) Log Message: ----------- Added version output when '-v' is used; added version output and doctest when wikipedia.py is called Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-08-01 21:37:41 UTC (rev 3936) +++ trunk/pywikipedia/wikipedia.py 2007-08-01 21:53:08 UTC (rev 3937) @@ -4203,6 +4203,9 @@ global logfile logfile = None elif arg == '-verbose' or arg == "-v": + import version + output('Pywikipediabot %s' % (version.getversion())) + output('Python %s' % (sys.version)) verbose += 1 else: # the argument is not global. Let the specific bot script care @@ -4648,3 +4651,11 @@ authhandler = urllib2.HTTPBasicAuthHandler(passman) authenticateURLopener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj),authhandler) urllib2.install_opener(authenticateURLopener) + +if __name__ == '__main__': + import version, doctest + print 'Pywikipediabot %s' % version.getversion() + print 'Python %s' % sys.version + doctest.testmod() + + \ No newline at end of file

16 years, 9 months

[Pywikipedia-l] SVN: [3936] trunk/pywikipedia/weblinkchecker.py

by wikipedian＠svn.wikimedia.org

Revision: 3936 Author: wikipedian Date: 2007-08-01 21:37:41 +0000 (Wed, 01 Aug 2007) Log Message: ----------- some servers redirect to .. although we are already in the root directory; ignore this. Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 21:19:26 UTC (rev 3935) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 21:37:41 UTC (rev 3936) @@ -270,9 +270,12 @@ # handle redirect to parent directory while redirTarget.startswith('../'): redirTarget = redirTarget[3:] - # change /foo/bar/ to /foo/ - directory = directory[:-1] - directory = directory[:directory.rindex('/') + 1] + # some servers redirect to .. although we are already + # in the root directory; ignore this. + if directory != '/': + # change /foo/bar/ to /foo/ + directory = directory[:-1] + directory = directory[:directory.rindex('/') + 1] self.changeUrl('%s://%s%s%s' % (self.protocol, self.host, directory, redirTarget)) return True else:

16 years, 9 months

[Pywikipedia-l] SVN: [3935] trunk/pywikipedia/version.py

by valhallasw＠svn.wikimedia.org

Revision: 3935 Author: valhallasw Date: 2007-08-01 21:19:26 +0000 (Wed, 01 Aug 2007) Log Message: ----------- New: version.py. returns pywikipedia version based on (in order) .svn/entries, version (file for use in nightlies) and wikipedia.__version__. Added Paths: ----------- trunk/pywikipedia/version.py Added: trunk/pywikipedia/version.py =================================================================== --- trunk/pywikipedia/version.py (rev 0) +++ trunk/pywikipedia/version.py 2007-08-01 21:19:26 UTC (rev 3935) @@ -0,0 +1,56 @@ +""" Module to determine the pywikipedia version (tag, revision and date) """ +# +# (C) Merlijn 'valhallasw' van Deen +# +# Distributed under the terms of the MIT license. +# +__version__ = '$Id$' + +import wikipediatools +import time +import sys + +class ParseError(Exception): + """ Parsing went wrong """ + +def getversion(): + try: + (tag, rev, date) = getversion_svn() + except Exception, e: + try: + (tag, rev, date) = getversion_nightly() + except IOError, e: + import wikipedia + d = wikipedia.__version__.split(' ') + tag = '' + date = time.strptime('T'.join(d[3:5]), '%Y-%m-%dT%H:%M:%SZ') + rev = d[2] + ' (wikipedia.py)' + + datestring = time.strftime('%b %d %Y, %H:%M:%S', date) + return '%s (r%s, %s)' % (tag, rev, datestring) + +def getversion_svn(): + entries = open(wikipediatools.absoluteFilename('.svn/entries')) + for i in range(4): + entries.readline() + tag = entries.readline().replace('svn+ssh://svn.wikimedia.org/svnroot/pywikipedia/', '').strip() + for i in range(4): + entries.readline() + date = time.strptime(entries.readline()[:19],'%Y-%m-%dT%H:%M:%S') + rev = entries.readline()[:-1] + if not date or not tag or not rev: + raise ParseError + return (tag, rev, date) + +def getversion_nightly(): + data = open(wikipediatools.absoluteFilename('version')) + tag = data.readline().strip() + date = time.strptime(data.readline()[:19],'%Y-%m-%dT%H:%M:%S') + rev = data.readline().strip() + if not date or not tag or not rev: + raise ParseError + return (tag, rev, date) + +if __name__ == '__main__': + print 'Pywikipedia %s' % getversion() + print 'Python %s' % sys.version \ No newline at end of file Property changes on: trunk/pywikipedia/version.py ___________________________________________________________________ Name: svn:keywords + Id *.c = svn:eol-style=native *.cpp = svn:eol-style=native *.h = svn:eol-style=native *.dsp = svn:eol-style=CRLF *.dsw = svn:eol-style=CRLF *.sh = svn:eol-style=native Name: svn:executable *.txt + svn:eol-style=native *.png = svn:mime-type=image/png *.jpg = svn:mime-type=image/jpeg Makefile = svn:eol-style=native Name: svn:eol-style + native

16 years, 9 months

[Pywikipedia-l] SVN: [3934] trunk/pywikipedia/wikipedia.py

by valhallasw＠svn.wikimedia.org

Revision: 3934 Author: valhallasw Date: 2007-08-01 19:53:05 +0000 (Wed, 01 Aug 2007) Log Message: ----------- Changed Site('de', 'wikipedia') comparison to 'wikipedia:de'. The latter is fine with wikipedia_family.py missing, the first is not. Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-08-01 16:37:37 UTC (rev 3933) +++ trunk/pywikipedia/wikipedia.py 2007-08-01 19:53:05 UTC (rev 3934) @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- """ Library to get and put pages on a MediaWiki. @@ -2850,7 +2850,7 @@ if site is None: site = getSite() - if site == Site('de', 'wikipedia'): + if site.sitename() == 'wikipedia:de': raise Error('The PyWikipediaBot is no longer allowed to touch categories on the German Wikipedia. See http://de.wikipedia.org/wiki/Hilfe_Diskussion:Personendaten/Archiv2#Positio…') s = categoryFormat(new, insite = site)

16 years, 9 months

[Pywikipedia-l] SVN: [3933] trunk/pywikipedia/weblinkchecker.py

by wikipedian＠svn.wikimedia.org

Revision: 3933 Author: wikipedian Date: 2007-08-01 16:37:37 +0000 (Wed, 01 Aug 2007) Log Message: ----------- fixed handling of URLs in multi-line templates, e.g. the infobox in http://de.wikipedia.org/wiki/Camon_%28Ari%C3%A8ge%29 Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 16:24:02 UTC (rev 3932) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 16:37:37 UTC (rev 3933) @@ -107,7 +107,7 @@ # might be a | or a } directly after a URL which does not belong to # the URL itself. # Blow up templates with spaces to avoid these problems. - templateWithParamsR = re.compile(r'{{(.*?[^ ])\|([^ ].*?)}}') + templateWithParamsR = re.compile(r'{{(.*?[^ ])\|([^ ].*?)}}', re.DOTALL) while templateWithParamsR.search(text): text = templateWithParamsR.sub(r'{{ \1 | \2 }}', text)

16 years, 9 months

[Pywikipedia-l] SVN: [3932] trunk/pywikipedia/weblinkchecker.py

by wikipedian＠svn.wikimedia.org

Revision: 3932 Author: wikipedian Date: 2007-08-01 16:24:02 +0000 (Wed, 01 Aug 2007) Log Message: ----------- don't crash when the Internet Archive gives a 403, e.g. on http://web.archive.org/web/*/http://highmarkfunds.stockpoint.com/highmarkfu… Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 16:17:20 UTC (rev 3931) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 16:24:02 UTC (rev 3932) @@ -147,7 +147,12 @@ def getArchiveURL(self): wikipedia.output(u'Consulting the Internet Archive for %s' % self.url) archiveURL = 'http://web.archive.org/web/*/%s' % self.url - f = urllib2.urlopen(archiveURL) + try: + f = urllib2.urlopen(archiveURL) + except urllib2.HTTPError: + # The Internet Archive yields a 403 error when the site was not + # archived due to robots.txt restrictions. + return None text = f.read() if text.find("Search Results for ") != -1: return archiveURL

16 years, 9 months

[Pywikipedia-l] SVN: [3931] trunk/pywikipedia/weblinkchecker.py

by wikipedian＠svn.wikimedia.org

Revision: 3931 Author: wikipedian Date: 2007-08-01 16:17:20 +0000 (Wed, 01 Aug 2007) Log Message: ----------- added -repeat parameter: Loads all wiki pages where dead links were found during a prior run Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 16:05:20 UTC (rev 3930) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 16:17:20 UTC (rev 3931) @@ -13,6 +13,9 @@ two times, with a time lag of at least one week. Such links will be logged to a .txt file in the deadlinks subdirectory. +After running the bot and waiting for at least one weak, you can re-check those +pages where dead links where found, using the -repeat parameter. + In addition to the logging step, it is possible to automatically report dead links to the talk page of the article where the link was found. To use this feature, set report_dead_links_on_talk = True in your user-config.py, or @@ -30,8 +33,14 @@ Loads all wiki pages using the Special:Allpages feature, starting at "Example page" + python weblinkchecker.py -weblink:www.example.org + Loads all wiki pages that link to www.example.org + python weblinkchecker.py Example page Only checks links found in the wiki page "Example page" + + python weblinkchecker.py -repeat + Loads all wiki pages where dead links were found during a prior run """ # @@ -571,6 +580,19 @@ thread.setDaemon(True) thread.start() +def RepeatPageGenerator(): + history = History(None) + pageTitles = set() + for (key, value) in history.historyDict.iteritems(): + for entry in value: + pageTitle = entry[0] + pageTitles.add(pageTitle) + pageTitles = list(pageTitles) + pageTitles.sort() + for pageTitle in pageTitles: + page = wikipedia.Page(wikipedia.getSite(), pageTitle) + yield page + def countLinkCheckThreads(): i = 0 for thread in threading.enumerate(): @@ -597,6 +619,8 @@ config.report_dead_links_on_talk = False elif arg.startswith('-namespace:'): namespaces.append(int(arg[11:])) + elif arg == '-repeat': + gen = RepeatPageGenerator() else: generator = genFactory.handleArg(arg) if generator:

16 years, 9 months

[Pywikipedia-l] SVN: [3930] trunk/pywikiparser

by valhallasw＠svn.wikimedia.org

Revision: 3930 Author: valhallasw Date: 2007-08-01 16:05:20 +0000 (Wed, 01 Aug 2007) Log Message: ----------- Some BufferedReader hacks; Parser updated to understand both simple wikilinks [[blah]] and simple templates {{blah}}. [[boo{{bah}}]] is also understood \o/ Modified Paths: -------------- trunk/pywikiparser/BufferedReader.py trunk/pywikiparser/Parser.py Modified: trunk/pywikiparser/BufferedReader.py =================================================================== --- trunk/pywikiparser/BufferedReader.py 2007-08-01 12:31:16 UTC (rev 3929) +++ trunk/pywikiparser/BufferedReader.py 2007-08-01 16:05:20 UTC (rev 3930) @@ -1,7 +1,41 @@ class BufferedReader(object): + """ Buffered reader. Usage: + + >>> reader = BufferedReader((i for i in range(10))) + >>> restore = reader.getrestore() + >>> restore + -1 + >>> reader.next() + 0 + >>> reader.next() + 1 + >>> reader.undo(-1) + >>> reader.next() + 0 + >>> restore = reader.commit(-1) + >>> restore + -1 + >>> reader.next() + 1 + >>> reader.getrestore() + 0 + >>> reader.next() + 2 + >>> reader.undo(0) + >>> reader.next() + 2 + >>> reader.commit(0) + 1 + >>> reader.undo(1) + >>> reader.next() + 3 + >>> reader.undo(-1) + >>> reader.next() + 1 + """ + def __init__(self, generator): - self.inbuffer = [] - self.outbuffer = [] + self.buffer = [] self.counter = -1 self.generator = generator self.gen = self._generator() @@ -31,33 +65,36 @@ def next(self, *args, **kwargs): return self.gen.next(*args, **kwargs) - def peek(self): - if len(self.outbuffer) <= self.counter+1: + def peek(self, num=1): + if len(self.buffer) <= self.counter+num: data = self.generator.next() - self.inbuffer.append(data) - self.outbuffer.append(data) - return self.outbuffer[self.counter+1] + self.buffer.append(data) + return self.buffer[self.counter+num] def _generator(self): while(True): self.counter += 1 - if len(self.outbuffer) <= self.counter: + if len(self.buffer) <= self.counter: data = self.generator.next() - self.inbuffer.append(data) - self.outbuffer.append(data) - yield self.outbuffer[self.counter] + self.buffer.append(data) + yield self.buffer[self.counter] - def commit(self): - self.inbuffer = self.inbuffer[self.counter+1:] - self.outbuffer = self.outbuffer[self.counter+1:] - self.counter = -1 + def getrestore(self): + return self.counter + + def commit(self, counter): + if counter == -1: + # clear memory + self.buffer = self.buffer[self.counter+1:] + self.counter = -1 + self.gen = self._generator() + return self.counter - def undo(self): - self.outbuffer = self.inbuffer[:] - self.counter = -1 + def undo(self, counter): + self.counter = counter self.gen = self._generator() - - def push(self, data): - self.outbuffer.append(data) - self.gen = self._generator() \ No newline at end of file + +if __name__ == "__main__": + import doctest + doctest.testmod() \ No newline at end of file Modified: trunk/pywikiparser/Parser.py =================================================================== --- trunk/pywikiparser/Parser.py 2007-08-01 12:31:16 UTC (rev 3929) +++ trunk/pywikiparser/Parser.py 2007-08-01 16:05:20 UTC (rev 3930) @@ -46,39 +46,41 @@ self.italic = False self.bold = False + restore = self.lex.getrestore() + try: while(True): token = self.lex.peek() if token[0] in breaktoken: break - node = self.parsetoken(token) + node = self.parsetoken(token, restore) print "Adding %r (was %r)" % (node,token) self.par.extend(node) - self.lex.commit() + restore = self.lex.commit(restore) except StopIteration: pass return self.root - def parsetoken(self, token): + def parsetoken(self, token, restore): # The function to call is parser<token> - exec("data = self.parse%s()" % token[0].name, globals(), locals()) + exec("data = self.parse%s(restore)" % token[0].name, globals(), locals()) return data - def parseEOF(self): + def parseEOF(self, restore): token = self.expect(Tokens.EOF) raise StopIteration # Special functions that directly access the storage tree - def parseNEWPAR(self): + def parseNEWPAR(self, restore): token = self.expect(Tokens.NEWPAR) self.par = self.root.appendElement('p') self.bold = False self.italic = False return [] - def parseAPOSTROPHE(self): + def parseAPOSTROPHE(self, restore): num = len(self.eat(Tokens.APOSTROPHE)) #prepare length @@ -126,46 +128,46 @@ # Functions that return the input directly - def parseSQRE_CLOSE(self): + def parseSQRE_CLOSE(self, restore): return self.expect(Tokens.SQRE_CLOSE) - def parsePIPE(self): + def parsePIPE(self, restore): return self.expect(Tokens.PIPE) - def parseEQUAL_SIGN(self): + def parseEQUAL_SIGN(self, restore): return self.expect(Tokens.EQUAL_SIGN) - def parseCURL_CLOSE(self): + def parseCURL_CLOSE(self, restore): return self.expect(Tokens.CURL_CLOSE) - def parseANGL_CLOSE(self): + def parseANGL_CLOSE(self, restore): return self.expect(Tokens.ANGL_CLOSE) - def parseASTERISK(self): + def parseASTERISK(self, restore): return self.expect(Tokens.ASTERISK) - def parseCOLON(self): + def parseCOLON(self, restore): return self.expect(Tokens.COLON) - def parseSEMICOLON(self): + def parseSEMICOLON(self, restore): return self.expect(Tokens.SEMICOLON) - def parseHASH(self): + def parseHASH(self, restore): return self.expect(Tokens.HASH) - def parseTAB_NEWLINE(self): + def parseTAB_NEWLINE(self, restore): return self.expect(Tokens.TAB_NEWLINE) - def parseTAB_CLOSE(self): + def parseTAB_CLOSE(self, restore): return self.expect(Tokens.TAB_CLOSE) # True parser callers - def parseWHITESPACE(self): + def parseWHITESPACE(self, restore): # Todo: - return self.parseTEXT() + return self.parseTEXT(restore) - def parseTEXT(self): + def parseTEXT(self, restore): text = self.eat([Tokens.TEXT, Tokens.WHITESPACE]) if text: @@ -173,49 +175,48 @@ else: return [] - def parseSQRE_OPEN(self): + def parseSQRE_OPEN(self, restore): try: return self.parseWikilink() except ParseError: pass - self.lex.undo() + self.lex.undo(restore) try: return self.parseExternallink() except ParseError: pass - self.lex.undo() + self.lex.undo(restore) return self.expect(Tokens.SQRE_OPEN) - def parseCURL_OPEN(self): + def parseCURL_OPEN(self, restore): try: return self.parseTemplateparam() except ParseError: pass - self.lex.undo() + self.lex.undo(restore) try: return self.parseTemplate() except ParseError: pass - self.lex.undo() + self.lex.undo(restore) return self.expect(Tokens.CURL_OPEN) - def parseANGL_OPEN(self): + def parseANGL_OPEN(self, restore): try: return self.parseHTML() except ParseError: pass - self.lex.undo() + self.lex.undo(restore) return self.expect(Tokens.ANGL_OPEN) - def parseTAB_OPEN(self): + def parseTAB_OPEN(self, restore): try: return self.parseWikitable() except ParseError: pass - self.lex.undo() + self.lex.undo(restore) return self.expect(Tokens.TAB_OPEN) - titlere = re.compile(r"[^\^\]#<>\[\|\{\}\n]*$") def parseWikilink(self): retval = dom.Element('') self.expect(Tokens.SQRE_OPEN) @@ -224,17 +225,53 @@ pre = self.eat(Tokens.SQRE_OPEN) if pre: retval.append(pre) - - title = self.eat(Tokens.TEXT) # temp. needs to allow templates etc. - - link = retval.appendElement('wikilink') - link.appendElement('url').append(title) - self.expect(Tokens.SQRE_CLOSE) + wikilink = retval.appendElement('wikilink') + # get page title + title = wikilink.appendElement('title') + + #parse title + title.extend(self.parseTitle(Tokens.SQRE_CLOSE)) + self.expect(Tokens.SQRE_CLOSE) - return retval - + self.expect(Tokens.SQRE_CLOSE) + return retval + + + +# while( titlere.match(next) ): +# title += next +# next = self.lex.peek() +# +# +# else: +# break +# while(True): +# param = .Element('parameter') +# parampiece = self.parse([Tokens.SQRE_CLOSE, Tokens.PIPE]) +# param.extend(parampiece) +# if (self.lex.peek( )[0] == Tokens.SQRE_CLOSE) and +# (self.lex.peek(2)[0] != Tokens.SQRE_CLOSE): # \][^\]]: a single ] +# param.append('[') +# continue +# else: +# break +# +# +# +# breaktoken = self.lex.peek() +# if breaktoken[0] == Tokens.PIPE: +# break +# elif breaktoken[0] == Tokens.SQRE_CLOSE: +# next = self.lex.peek(2) +# if next[0] == Tokens.SQRE_CLOSE: +# +# self.expect(Tokens.SQRE_CLOSE) +# self.expect(Tokens.SQRE_CLOSE) +# return retval +# + def parseExternallink(self): raise ParseError("Needs implementation") @@ -242,11 +279,52 @@ raise ParseError("Needs implementation") def parseTemplate(self): - raise ParseError("Needs implementation") + retval = dom.Element('') + self.expect(Tokens.CURL_OPEN) + self.expect(Tokens.CURL_OPEN) + pre = self.eat(Tokens.CURL_OPEN) + print 'pre: ' + pre + if pre: + retval.append(pre) + + wikilink = retval.appendElement('template') + # get page title + title = wikilink.appendElement('title') + title.extend(self.parseTitle(Tokens.CURL_CLOSE)) + + self.expect(Tokens.CURL_CLOSE) + self.expect(Tokens.CURL_CLOSE) + + return retval + def parseHTML(self): raise ParseError("Needs implementation") def parseWikitable(self): raise ParseError("Needs implementation") + + titlere = re.compile(r"[^\^\]<>\[\|\{\}\n]*$") + def parseTitle(self, closetoken): + title = dom.Element('title') + while(True): + next = self.lex.peek() + if next[0] == closetoken or next[0] == Tokens.PIPE: + break + elif next[0] == Tokens.CURL_OPEN: # allow templates to expand + restore = self.lex.getrestore() + data = self.parseCURL_OPEN(restore) + print 'Parsed template: %r' % (data,) + for item in data: + if isinstance(item, basestring): + if not self.titlere.match(item): + raise ParseError('illegal wiki link') + title.extend(data) + else: + next = self.lex.next() + if not self.titlere.match(next[1]): + raise ParseError('illegal wiki link') + title.append(next[1]) + return title + \ No newline at end of file

16 years, 9 months

[Pywikipedia-l] SVN: [3929] trunk/pywikiparser

by valhallasw＠svn.wikimedia.org

Revision: 3929 Author: valhallasw Date: 2007-08-01 12:31:16 +0000 (Wed, 01 Aug 2007) Log Message: ----------- Lexer.py: special characters now return one token; all tokens have text representation attached. Parser.py: updated to allow for new Lexer Modified Paths: -------------- trunk/pywikiparser/Lexer.py trunk/pywikiparser/Parser.py Modified: trunk/pywikiparser/Lexer.py =================================================================== --- trunk/pywikiparser/Lexer.py 2007-08-01 00:30:33 UTC (rev 3928) +++ trunk/pywikiparser/Lexer.py 2007-08-01 12:31:16 UTC (rev 3929) @@ -45,11 +45,16 @@ class Lexer: """ Lexer class for mediawiki wikitext. Used by the Parser module + Lexer.lexer() returns a generator that returns (Token, text) pairs. The text represents the actual text data, the token the interpreted data. - >>> l = Lexer('Test with [[wikilink|description]], {{template|parameter\\'s|{{nested}}=booh}}, \n\n new paragraphs, <html>, {| tables |- |}') + >>> l = Lexer('Test with [[wikilink|description]], {{template|parameter\\'s|{{nested}}=booh}}, \\n\\n new paragraphs, <html>, {| tables |- |}') >>> gen = l.lexer() - >>> [token for token in gen] - [(258, 'Test'), (272, ' '), (258, 'with'), (272, ' '), (259, 2), (258, 'wikilink'), (261, None), (258, 'description'), (260, 2), (258, ','), (272, ' '), (264, 2), (258, 'template'), (261, None), (258, 'parameter'), (263, 1), (258, 's'), (261, None), (264, 2), (258, 'nested'), (265, 2), (262, 1), (258, 'booh'), (265, 2), (258, ','), (268, ' \n\n '), (258, 'new'), (272, ' '), (258, 'paragraphs,'), (272, ' '), (266, 1), (258, 'html'), (267, 1), (258, ','), (272, ' '), (264, 1), (261, None), (272, ' '), (258, 'tables'), (272, ' '), (270, None), (258, '-'), (271, None), (273, None)] + >>> gen.next() + (<T_TEXT>, 'Test') + >>> gen.next() + (<T_WHITESPACE>, ' ') + >>> [token for token in gen][:10] + [(<T_TEXT>, 'with'), (<T_WHITESPACE>, ' '), (<T_SQRE_OPEN>, '['), (<T_SQRE_OPEN>, '['), (<T_TEXT>, 'wikilink'), (<T_PIPE>, None), (<T_TEXT>, 'description'), (<T_SQRE_CLOSE>, ']'), (<T_SQRE_CLOSE>, ']'), (<T_TEXT>, ',')] """ def __init__(self, string): @@ -60,54 +65,50 @@ try: c = self.getchar() while True: - if (c in ('[', ']', '{', '}', '<', '>', '=', '\'', '*', ':', ';', '#')): + if (c in ('[', ']', '}', '<', '>', '=', '\'', '*', ':', ';', '#')): if text: yield (Tokens.TEXT, text) text = '' - num = 1 - try: - t = self.getchar() - while (t == c): - num += 1 - t = self.getchar() - - finally: - if (c == '['): yield (Tokens.SQRE_OPEN, num) - elif (c == ']'): yield (Tokens.SQRE_CLOSE, num) - elif (c == '{'): yield (Tokens.CURL_OPEN, num) - elif (c == '}'): yield (Tokens.CURL_CLOSE, num) - elif (c == '<'): yield (Tokens.ANGL_OPEN, num) - elif (c == '>'): yield (Tokens.ANGL_CLOSE, num) - elif (c == '='): yield (Tokens.EQUAL_SIGN, num) - elif (c == '\''): yield(Tokens.APOSTROPHE, num) - elif (c == '*'): yield (Tokens.ASTERISK, num) - elif (c == ':'): yield (Tokens.COLON, num) - elif (c == ';'): yield (Tokens.SEMICOLON, num) - elif (c == '#'): yield (Tokens.HASH, num) + + if (c == '['): yield (Tokens.SQRE_OPEN, c) + elif (c == ']'): yield (Tokens.SQRE_CLOSE, c) + elif (c == '}'): yield (Tokens.CURL_CLOSE, c) + elif (c == '<'): yield (Tokens.ANGL_OPEN, c) + elif (c == '>'): yield (Tokens.ANGL_CLOSE, c) + elif (c == '='): yield (Tokens.EQUAL_SIGN, c) + elif (c == '\''): yield(Tokens.APOSTROPHE, c) + elif (c == '*'): yield (Tokens.ASTERISK, c) + elif (c == ':'): yield (Tokens.COLON, c) + elif (c == ';'): yield (Tokens.SEMICOLON, c) + elif (c == '#'): yield (Tokens.HASH, c) + c = self.getchar() + elif (c == '{'): + if text: + yield (Tokens.TEXT, text) + text = '' + t = self.getchar() + if (t == '|'): + yield (Tokens.TAB_OPEN, '{|') + c = self.getchar() + else: + yield (Tokens.CURL_OPEN, '{') + c = t elif (c == '|'): if text: yield (Tokens.TEXT, text) text = '' - try: - t = self.getchar() - except StopIteration: - yield (Tokens.PIPE, None) - raise + t = self.getchar() if (t == '-'): - yield (Tokens.TAB_NEWLINE, None) + yield (Tokens.TAB_NEWLINE, '|-') c = self.getchar() elif (t == '}'): - yield (Tokens.TAB_CLOSE, None) + yield (Tokens.TAB_CLOSE, '|}') c = self.getchar() else: - num = 1 - while (t == c): - num += 1 - t = self.getchar() - yield (Tokens.PIPE, num) - c = t + yield (Tokens.PIPE, None) + c = t elif re.match('\s', c): # whitespace eater pro (TM) if text: yield (Tokens.TEXT, text) @@ -131,4 +132,8 @@ yield (Tokens.EOF, None) def getchar(self): - return self.data.next() \ No newline at end of file + return self.data.next() + +if __name__ == "__main__": + import doctest + doctest.testmod() \ No newline at end of file Modified: trunk/pywikiparser/Parser.py =================================================================== --- trunk/pywikiparser/Parser.py 2007-08-01 00:30:33 UTC (rev 3928) +++ trunk/pywikiparser/Parser.py 2007-08-01 12:31:16 UTC (rev 3929) @@ -28,26 +28,24 @@ data = self.lex.peek() if data[0] in tokens: - return self.lex.next() + return self.lex.next()[1] else: raise ParseError('%r is not one of %r' % (data[0], tokens)) - - def expecttext(self): - data = self.lex.peek() - if data[0] in [Tokens.TEXT, Tokens.WHITESPACE]: - return self.lex.next() - elif data[0] in [Tokens.EQUAL_SIGN, Tokens.APOSTROPHE, Tokens.ASTERISK, - Tokens.COLON, Tokens.SEMICOLON, Tokens.HASH]: - data = self.lex.next() - return (data[0], data[0].__doc__[0]*data[1]) - else: - raise ParseError('%r is not parsable as text data' % (data[0],)) + def eat(self, tokens): + data = '' + try: + while(True): + data += self.expect(tokens) + except ParseError: + return data + def parse(self, breaktoken=[]): self.root = dom.Element('wikipage') self.par = self.root.appendElement('p') self.italic = False self.bold = False + try: while(True): token = self.lex.peek() @@ -61,7 +59,7 @@ except StopIteration: pass return self.root - + def parsetoken(self, token): # The function to call is parser<token> exec("data = self.parse%s()" % token[0].name, globals(), locals()) @@ -81,8 +79,7 @@ return [] def parseAPOSTROPHE(self): - token = self.expect(Tokens.APOSTROPHE) - num = token[1] + num = len(self.eat(Tokens.APOSTROPHE)) #prepare length if (num == 1): @@ -130,51 +127,38 @@ # Functions that return the input directly def parseSQRE_CLOSE(self): - token = self.expect(Tokens.SQRE_CLOSE) - return [']'*token[1]] + return self.expect(Tokens.SQRE_CLOSE) def parsePIPE(self): - token = self.expect(Tokens.PIPE) - return ['|'*token[1]] + return self.expect(Tokens.PIPE) def parseEQUAL_SIGN(self): - token = self.expect(Tokens.EQUAL_SIGN) - return ['='*token[1]] + return self.expect(Tokens.EQUAL_SIGN) def parseCURL_CLOSE(self): - token = self.expect(Tokens.CURL_CLOSE) - return ['}'*token[1]] + return self.expect(Tokens.CURL_CLOSE) def parseANGL_CLOSE(self): - token = self.expect(Tokens.ANGL_CLOSE) - return ['>'*token[1]] + return self.expect(Tokens.ANGL_CLOSE) def parseASTERISK(self): - token = self.expect(Tokens.ASTERISK) - return ['*'*token[1]] + return self.expect(Tokens.ASTERISK) def parseCOLON(self): - token = self.expect(Tokens.COLON) - return [':'*token[1]] + return self.expect(Tokens.COLON) def parseSEMICOLON(self): - token = self.expect(Tokens.SEMICOLON) - return [';'*token[1]] + return self.expect(Tokens.SEMICOLON) def parseHASH(self): - token = self.expect(Tokens.HASH) - return ['#'*token[1]] + return self.expect(Tokens.HASH) def parseTAB_NEWLINE(self): - token = self.expect(Tokens.TAB_NEWLINE) - return ['|-'] + return self.expect(Tokens.TAB_NEWLINE) def parseTAB_CLOSE(self): - token = self.expect(Tokens.TAB_CLOSE) - return ['|}'] - - - + return self.expect(Tokens.TAB_CLOSE) + # True parser callers def parseWHITESPACE(self): @@ -182,11 +166,7 @@ return self.parseTEXT() def parseTEXT(self): - text = '' - while(True): - try: - text += self.expect([Tokens.TEXT, Tokens.WHITESPACE])[1] - except ParseError: break + text = self.eat([Tokens.TEXT, Tokens.WHITESPACE]) if text: return [text] @@ -204,8 +184,7 @@ except ParseError: pass self.lex.undo() - token = self.expect(Tokens.SQRE_OPEN) - return ['['*token[1]] + return self.expect(Tokens.SQRE_OPEN) def parseCURL_OPEN(self): try: @@ -218,8 +197,7 @@ except ParseError: pass self.lex.undo() - token = self.expect(Tokens.CURL_OPEN) - return ['{'*token[1]] + return self.expect(Tokens.CURL_OPEN) def parseANGL_OPEN(self): try: @@ -227,8 +205,7 @@ except ParseError: pass self.lex.undo() - token = self.expect(Tokens.ANGL_OPEN) - return ['<'*token[1]] + return self.expect(Tokens.ANGL_OPEN) def parseTAB_OPEN(self): try: @@ -236,39 +213,25 @@ except ParseError: pass self.lex.undo() - token = self.expect(Tokens.TAB_OPEN) - return ['{|'] + return self.expect(Tokens.TAB_OPEN) titlere = re.compile(r"[^\^\]#<>\[\|\{\}\n]*$") def parseWikilink(self): retval = dom.Element('') - pre = self.expect(Tokens.SQRE_OPEN)[1]-2 + self.expect(Tokens.SQRE_OPEN) + self.expect(Tokens.SQRE_OPEN) - if pre < 0: - raise ParseError("Not enough opening brackets") - elif pre > 0: - retval.append('['*pre) + pre = self.eat(Tokens.SQRE_OPEN) + if pre: + retval.append(pre) - title = '' - while(True): - try: - data = self.expecttext()[1] - print data - except ParseError: break - if not self.titlere.match(data): - raise ParseError("Illegal page title") - else: - title += data + title = self.eat(Tokens.TEXT) # temp. needs to allow templates etc. link = retval.appendElement('wikilink') link.appendElement('url').append(title) - - aft = self.expect(Tokens.SQRE_CLOSE)[1]-2 - if aft < 0: - raise ParseError("Not enough closing brackets") - elif aft > 0: - self.lex.push((Tokens.SQRE_CLOSE, aft)) - + + self.expect(Tokens.SQRE_CLOSE) + self.expect(Tokens.SQRE_CLOSE) return retval

16 years, 9 months

[Pywikipedia-l] SVN: [3928] trunk/pywikipedia/weblinkchecker.py

by wikipedian＠svn.wikimedia.org

Revision: 3928 Author: wikipedian Date: 2007-08-01 00:30:33 +0000 (Wed, 01 Aug 2007) Log Message: ----------- made it possible to use all the typical parameters such as -ref:, -links:, -file:, and -weblink:. added -namespace: parameter. Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-07-31 16:03:31 UTC (rev 3927) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 00:30:33 UTC (rev 3928) @@ -9,7 +9,7 @@ The bot will store all links found dead in a .dat file in the deadlinks subdirectory. To avoid the removing of links which are only temporarily -unavailable, the bot only reports links which were reported dead at least +unavailable, the bot ONLY reports links which were reported dead at least two times, with a time lag of at least one week. Such links will be logged to a .txt file in the deadlinks subdirectory. @@ -29,7 +29,7 @@ python weblinkchecker.py -start:Example_page Loads all wiki pages using the Special:Allpages feature, starting at "Example page" - + python weblinkchecker.py Example page Only checks links found in the wiki page "Example page" """ @@ -91,22 +91,6 @@ re.compile('.*[\./(a)]berlinonline.de(/.*)?'), # a de: user wants to fix them by hand and doesn't want them to be deleted, see [[de:Benutzer:BLueFiSH.as/BZ]]. ] -class Global(object): - talk = config.report_dead_links_on_talk - - def handleArgs(self, args): - unhandledArguments = [] - for arg in args: - if arg == '-talk': - self.talk = True - elif arg == '-notalk': - self.talk = False - else: - unhandledArguments.append(arg) - return unhandledArguments - -globalvar = Global() - def weblinksIn(text, withoutBracketed = False, onlyBracketed = False): text = wikipedia.removeDisabledParts(text) @@ -546,10 +530,9 @@ Robot which will use several LinkCheckThreads at once to search for dead weblinks on pages provided by the given generator. ''' - def __init__(self, generator, start ='!'): + def __init__(self, generator): self.generator = generator - self.start = start - if globalvar.talk: + if config.report_dead_links_on_talk: #wikipedia.output("Starting talk page thread") reportThread = DeadLinkReportThread() # thread dies when program terminates @@ -598,25 +581,37 @@ def main(): gen = None - start = '!' - pageTitle = [] - args = wikipedia.handleArgs() - args = globalvar.handleArgs(args) - - for arg in args: - if arg.startswith('-start:'): - start = arg[7:] + singlePageTitle = [] + # Which namespaces should be processed? + # default to [] which means all namespaces will be processed + namespaces = [] + # This factory is responsible for processing command line arguments + # that are also used by other scripts and that determine on which pages + # to work on. + genFactory = pagegenerators.GeneratorFactory() + + for arg in wikipedia.handleArgs(): + if arg == '-talk': + config.report_dead_links_on_talk = True + elif arg == '-notalk': + config.report_dead_links_on_talk = False + elif arg.startswith('-namespace:'): + namespaces.append(int(arg[11:])) else: - pageTitle.append(arg) + generator = genFactory.handleArg(arg) + if generator: + gen = generator + else: + singlePageTitle.append(arg) - if pageTitle: - pageTitle = ' '.join(pageTitle) - page = wikipedia.Page(wikipedia.getSite(), pageTitle) + if singlePageTitle: + singlePageTitle = ' '.join(singlePageTitle) + page = wikipedia.Page(wikipedia.getSite(), singlePageTitle) gen = iter([page]) - else: - gen = pagegenerators.AllpagesPageGenerator(start) if gen: + if namespaces != []: + gen = pagegenerators.NamespaceFilterPageGenerator(gen, namespaces) gen = pagegenerators.PreloadingGenerator(gen, pageNumber = 240) gen = pagegenerators.RedirectFilterPageGenerator(gen) bot = WeblinkCheckerRobot(gen) @@ -651,7 +646,7 @@ bot.history.save() else: wikipedia.showHelp() - + if __name__ == "__main__": try: main()

16 years, 10 months

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

pywikibot