Pywikipedia-l

pywikipedia-l@lists.wikimedia.org

6393 discussions

SVN: [3932] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 01 Aug '07

01 Aug '07

Revision: 3932 Author: wikipedian Date: 2007-08-01 16:24:02 +0000 (Wed, 01 Aug 2007) Log Message: ----------- don't crash when the Internet Archive gives a 403, e.g. on http://web.archive.org/web/*/http://highmarkfunds.stockpoint.com/highmarkfu… Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 16:17:20 UTC (rev 3931) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 16:24:02 UTC (rev 3932) @@ -147,7 +147,12 @@ def getArchiveURL(self): wikipedia.output(u'Consulting the Internet Archive for %s' % self.url) archiveURL = 'http://web.archive.org/web/*/%s' % self.url - f = urllib2.urlopen(archiveURL) + try: + f = urllib2.urlopen(archiveURL) + except urllib2.HTTPError: + # The Internet Archive yields a 403 error when the site was not + # archived due to robots.txt restrictions. + return None text = f.read() if text.find("Search Results for ") != -1: return archiveURL

1 0

SVN: [3931] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 01 Aug '07

01 Aug '07

Revision: 3931 Author: wikipedian Date: 2007-08-01 16:17:20 +0000 (Wed, 01 Aug 2007) Log Message: ----------- added -repeat parameter: Loads all wiki pages where dead links were found during a prior run Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-08-01 16:05:20 UTC (rev 3930) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 16:17:20 UTC (rev 3931) @@ -13,6 +13,9 @@ two times, with a time lag of at least one week. Such links will be logged to a .txt file in the deadlinks subdirectory. +After running the bot and waiting for at least one weak, you can re-check those +pages where dead links where found, using the -repeat parameter. + In addition to the logging step, it is possible to automatically report dead links to the talk page of the article where the link was found. To use this feature, set report_dead_links_on_talk = True in your user-config.py, or @@ -30,8 +33,14 @@ Loads all wiki pages using the Special:Allpages feature, starting at "Example page" + python weblinkchecker.py -weblink:www.example.org + Loads all wiki pages that link to www.example.org + python weblinkchecker.py Example page Only checks links found in the wiki page "Example page" + + python weblinkchecker.py -repeat + Loads all wiki pages where dead links were found during a prior run """ # @@ -571,6 +580,19 @@ thread.setDaemon(True) thread.start() +def RepeatPageGenerator(): + history = History(None) + pageTitles = set() + for (key, value) in history.historyDict.iteritems(): + for entry in value: + pageTitle = entry[0] + pageTitles.add(pageTitle) + pageTitles = list(pageTitles) + pageTitles.sort() + for pageTitle in pageTitles: + page = wikipedia.Page(wikipedia.getSite(), pageTitle) + yield page + def countLinkCheckThreads(): i = 0 for thread in threading.enumerate(): @@ -597,6 +619,8 @@ config.report_dead_links_on_talk = False elif arg.startswith('-namespace:'): namespaces.append(int(arg[11:])) + elif arg == '-repeat': + gen = RepeatPageGenerator() else: generator = genFactory.handleArg(arg) if generator:

1 0

SVN: [3930] trunk/pywikiparser
by valhallasw＠svn.wikimedia.org 01 Aug '07

01 Aug '07

Revision: 3930 Author: valhallasw Date: 2007-08-01 16:05:20 +0000 (Wed, 01 Aug 2007) Log Message: ----------- Some BufferedReader hacks; Parser updated to understand both simple wikilinks [[blah]] and simple templates {{blah}}. [[boo{{bah}}]] is also understood \o/ Modified Paths: -------------- trunk/pywikiparser/BufferedReader.py trunk/pywikiparser/Parser.py Modified: trunk/pywikiparser/BufferedReader.py =================================================================== --- trunk/pywikiparser/BufferedReader.py 2007-08-01 12:31:16 UTC (rev 3929) +++ trunk/pywikiparser/BufferedReader.py 2007-08-01 16:05:20 UTC (rev 3930) @@ -1,7 +1,41 @@ class BufferedReader(object): + """ Buffered reader. Usage: + + >>> reader = BufferedReader((i for i in range(10))) + >>> restore = reader.getrestore() + >>> restore + -1 + >>> reader.next() + 0 + >>> reader.next() + 1 + >>> reader.undo(-1) + >>> reader.next() + 0 + >>> restore = reader.commit(-1) + >>> restore + -1 + >>> reader.next() + 1 + >>> reader.getrestore() + 0 + >>> reader.next() + 2 + >>> reader.undo(0) + >>> reader.next() + 2 + >>> reader.commit(0) + 1 + >>> reader.undo(1) + >>> reader.next() + 3 + >>> reader.undo(-1) + >>> reader.next() + 1 + """ + def __init__(self, generator): - self.inbuffer = [] - self.outbuffer = [] + self.buffer = [] self.counter = -1 self.generator = generator self.gen = self._generator() @@ -31,33 +65,36 @@ def next(self, *args, **kwargs): return self.gen.next(*args, **kwargs) - def peek(self): - if len(self.outbuffer) <= self.counter+1: + def peek(self, num=1): + if len(self.buffer) <= self.counter+num: data = self.generator.next() - self.inbuffer.append(data) - self.outbuffer.append(data) - return self.outbuffer[self.counter+1] + self.buffer.append(data) + return self.buffer[self.counter+num] def _generator(self): while(True): self.counter += 1 - if len(self.outbuffer) <= self.counter: + if len(self.buffer) <= self.counter: data = self.generator.next() - self.inbuffer.append(data) - self.outbuffer.append(data) - yield self.outbuffer[self.counter] + self.buffer.append(data) + yield self.buffer[self.counter] - def commit(self): - self.inbuffer = self.inbuffer[self.counter+1:] - self.outbuffer = self.outbuffer[self.counter+1:] - self.counter = -1 + def getrestore(self): + return self.counter + + def commit(self, counter): + if counter == -1: + # clear memory + self.buffer = self.buffer[self.counter+1:] + self.counter = -1 + self.gen = self._generator() + return self.counter - def undo(self): - self.outbuffer = self.inbuffer[:] - self.counter = -1 + def undo(self, counter): + self.counter = counter self.gen = self._generator() - - def push(self, data): - self.outbuffer.append(data) - self.gen = self._generator() \ No newline at end of file + +if __name__ == "__main__": + import doctest + doctest.testmod() \ No newline at end of file Modified: trunk/pywikiparser/Parser.py =================================================================== --- trunk/pywikiparser/Parser.py 2007-08-01 12:31:16 UTC (rev 3929) +++ trunk/pywikiparser/Parser.py 2007-08-01 16:05:20 UTC (rev 3930) @@ -46,39 +46,41 @@ self.italic = False self.bold = False + restore = self.lex.getrestore() + try: while(True): token = self.lex.peek() if token[0] in breaktoken: break - node = self.parsetoken(token) + node = self.parsetoken(token, restore) print "Adding %r (was %r)" % (node,token) self.par.extend(node) - self.lex.commit() + restore = self.lex.commit(restore) except StopIteration: pass return self.root - def parsetoken(self, token): + def parsetoken(self, token, restore): # The function to call is parser<token> - exec("data = self.parse%s()" % token[0].name, globals(), locals()) + exec("data = self.parse%s(restore)" % token[0].name, globals(), locals()) return data - def parseEOF(self): + def parseEOF(self, restore): token = self.expect(Tokens.EOF) raise StopIteration # Special functions that directly access the storage tree - def parseNEWPAR(self): + def parseNEWPAR(self, restore): token = self.expect(Tokens.NEWPAR) self.par = self.root.appendElement('p') self.bold = False self.italic = False return [] - def parseAPOSTROPHE(self): + def parseAPOSTROPHE(self, restore): num = len(self.eat(Tokens.APOSTROPHE)) #prepare length @@ -126,46 +128,46 @@ # Functions that return the input directly - def parseSQRE_CLOSE(self): + def parseSQRE_CLOSE(self, restore): return self.expect(Tokens.SQRE_CLOSE) - def parsePIPE(self): + def parsePIPE(self, restore): return self.expect(Tokens.PIPE) - def parseEQUAL_SIGN(self): + def parseEQUAL_SIGN(self, restore): return self.expect(Tokens.EQUAL_SIGN) - def parseCURL_CLOSE(self): + def parseCURL_CLOSE(self, restore): return self.expect(Tokens.CURL_CLOSE) - def parseANGL_CLOSE(self): + def parseANGL_CLOSE(self, restore): return self.expect(Tokens.ANGL_CLOSE) - def parseASTERISK(self): + def parseASTERISK(self, restore): return self.expect(Tokens.ASTERISK) - def parseCOLON(self): + def parseCOLON(self, restore): return self.expect(Tokens.COLON) - def parseSEMICOLON(self): + def parseSEMICOLON(self, restore): return self.expect(Tokens.SEMICOLON) - def parseHASH(self): + def parseHASH(self, restore): return self.expect(Tokens.HASH) - def parseTAB_NEWLINE(self): + def parseTAB_NEWLINE(self, restore): return self.expect(Tokens.TAB_NEWLINE) - def parseTAB_CLOSE(self): + def parseTAB_CLOSE(self, restore): return self.expect(Tokens.TAB_CLOSE) # True parser callers - def parseWHITESPACE(self): + def parseWHITESPACE(self, restore): # Todo: - return self.parseTEXT() + return self.parseTEXT(restore) - def parseTEXT(self): + def parseTEXT(self, restore): text = self.eat([Tokens.TEXT, Tokens.WHITESPACE]) if text: @@ -173,49 +175,48 @@ else: return [] - def parseSQRE_OPEN(self): + def parseSQRE_OPEN(self, restore): try: return self.parseWikilink() except ParseError: pass - self.lex.undo() + self.lex.undo(restore) try: return self.parseExternallink() except ParseError: pass - self.lex.undo() + self.lex.undo(restore) return self.expect(Tokens.SQRE_OPEN) - def parseCURL_OPEN(self): + def parseCURL_OPEN(self, restore): try: return self.parseTemplateparam() except ParseError: pass - self.lex.undo() + self.lex.undo(restore) try: return self.parseTemplate() except ParseError: pass - self.lex.undo() + self.lex.undo(restore) return self.expect(Tokens.CURL_OPEN) - def parseANGL_OPEN(self): + def parseANGL_OPEN(self, restore): try: return self.parseHTML() except ParseError: pass - self.lex.undo() + self.lex.undo(restore) return self.expect(Tokens.ANGL_OPEN) - def parseTAB_OPEN(self): + def parseTAB_OPEN(self, restore): try: return self.parseWikitable() except ParseError: pass - self.lex.undo() + self.lex.undo(restore) return self.expect(Tokens.TAB_OPEN) - titlere = re.compile(r"[^\^\]#<>\[\|\{\}\n]*$") def parseWikilink(self): retval = dom.Element('') self.expect(Tokens.SQRE_OPEN) @@ -224,17 +225,53 @@ pre = self.eat(Tokens.SQRE_OPEN) if pre: retval.append(pre) - - title = self.eat(Tokens.TEXT) # temp. needs to allow templates etc. - - link = retval.appendElement('wikilink') - link.appendElement('url').append(title) - self.expect(Tokens.SQRE_CLOSE) + wikilink = retval.appendElement('wikilink') + # get page title + title = wikilink.appendElement('title') + + #parse title + title.extend(self.parseTitle(Tokens.SQRE_CLOSE)) + self.expect(Tokens.SQRE_CLOSE) - return retval - + self.expect(Tokens.SQRE_CLOSE) + return retval + + + +# while( titlere.match(next) ): +# title += next +# next = self.lex.peek() +# +# +# else: +# break +# while(True): +# param = .Element('parameter') +# parampiece = self.parse([Tokens.SQRE_CLOSE, Tokens.PIPE]) +# param.extend(parampiece) +# if (self.lex.peek( )[0] == Tokens.SQRE_CLOSE) and +# (self.lex.peek(2)[0] != Tokens.SQRE_CLOSE): # \][^\]]: a single ] +# param.append('[') +# continue +# else: +# break +# +# +# +# breaktoken = self.lex.peek() +# if breaktoken[0] == Tokens.PIPE: +# break +# elif breaktoken[0] == Tokens.SQRE_CLOSE: +# next = self.lex.peek(2) +# if next[0] == Tokens.SQRE_CLOSE: +# +# self.expect(Tokens.SQRE_CLOSE) +# self.expect(Tokens.SQRE_CLOSE) +# return retval +# + def parseExternallink(self): raise ParseError("Needs implementation") @@ -242,11 +279,52 @@ raise ParseError("Needs implementation") def parseTemplate(self): - raise ParseError("Needs implementation") + retval = dom.Element('') + self.expect(Tokens.CURL_OPEN) + self.expect(Tokens.CURL_OPEN) + pre = self.eat(Tokens.CURL_OPEN) + print 'pre: ' + pre + if pre: + retval.append(pre) + + wikilink = retval.appendElement('template') + # get page title + title = wikilink.appendElement('title') + title.extend(self.parseTitle(Tokens.CURL_CLOSE)) + + self.expect(Tokens.CURL_CLOSE) + self.expect(Tokens.CURL_CLOSE) + + return retval + def parseHTML(self): raise ParseError("Needs implementation") def parseWikitable(self): raise ParseError("Needs implementation") + + titlere = re.compile(r"[^\^\]<>\[\|\{\}\n]*$") + def parseTitle(self, closetoken): + title = dom.Element('title') + while(True): + next = self.lex.peek() + if next[0] == closetoken or next[0] == Tokens.PIPE: + break + elif next[0] == Tokens.CURL_OPEN: # allow templates to expand + restore = self.lex.getrestore() + data = self.parseCURL_OPEN(restore) + print 'Parsed template: %r' % (data,) + for item in data: + if isinstance(item, basestring): + if not self.titlere.match(item): + raise ParseError('illegal wiki link') + title.extend(data) + else: + next = self.lex.next() + if not self.titlere.match(next[1]): + raise ParseError('illegal wiki link') + title.append(next[1]) + return title + \ No newline at end of file

1 0

SVN: [3929] trunk/pywikiparser
by valhallasw＠svn.wikimedia.org 01 Aug '07

01 Aug '07

Revision: 3929 Author: valhallasw Date: 2007-08-01 12:31:16 +0000 (Wed, 01 Aug 2007) Log Message: ----------- Lexer.py: special characters now return one token; all tokens have text representation attached. Parser.py: updated to allow for new Lexer Modified Paths: -------------- trunk/pywikiparser/Lexer.py trunk/pywikiparser/Parser.py Modified: trunk/pywikiparser/Lexer.py =================================================================== --- trunk/pywikiparser/Lexer.py 2007-08-01 00:30:33 UTC (rev 3928) +++ trunk/pywikiparser/Lexer.py 2007-08-01 12:31:16 UTC (rev 3929) @@ -45,11 +45,16 @@ class Lexer: """ Lexer class for mediawiki wikitext. Used by the Parser module + Lexer.lexer() returns a generator that returns (Token, text) pairs. The text represents the actual text data, the token the interpreted data. - >>> l = Lexer('Test with [[wikilink|description]], {{template|parameter\\'s|{{nested}}=booh}}, \n\n new paragraphs, <html>, {| tables |- |}') + >>> l = Lexer('Test with [[wikilink|description]], {{template|parameter\\'s|{{nested}}=booh}}, \\n\\n new paragraphs, <html>, {| tables |- |}') >>> gen = l.lexer() - >>> [token for token in gen] - [(258, 'Test'), (272, ' '), (258, 'with'), (272, ' '), (259, 2), (258, 'wikilink'), (261, None), (258, 'description'), (260, 2), (258, ','), (272, ' '), (264, 2), (258, 'template'), (261, None), (258, 'parameter'), (263, 1), (258, 's'), (261, None), (264, 2), (258, 'nested'), (265, 2), (262, 1), (258, 'booh'), (265, 2), (258, ','), (268, ' \n\n '), (258, 'new'), (272, ' '), (258, 'paragraphs,'), (272, ' '), (266, 1), (258, 'html'), (267, 1), (258, ','), (272, ' '), (264, 1), (261, None), (272, ' '), (258, 'tables'), (272, ' '), (270, None), (258, '-'), (271, None), (273, None)] + >>> gen.next() + (<T_TEXT>, 'Test') + >>> gen.next() + (<T_WHITESPACE>, ' ') + >>> [token for token in gen][:10] + [(<T_TEXT>, 'with'), (<T_WHITESPACE>, ' '), (<T_SQRE_OPEN>, '['), (<T_SQRE_OPEN>, '['), (<T_TEXT>, 'wikilink'), (<T_PIPE>, None), (<T_TEXT>, 'description'), (<T_SQRE_CLOSE>, ']'), (<T_SQRE_CLOSE>, ']'), (<T_TEXT>, ',')] """ def __init__(self, string): @@ -60,54 +65,50 @@ try: c = self.getchar() while True: - if (c in ('[', ']', '{', '}', '<', '>', '=', '\'', '*', ':', ';', '#')): + if (c in ('[', ']', '}', '<', '>', '=', '\'', '*', ':', ';', '#')): if text: yield (Tokens.TEXT, text) text = '' - num = 1 - try: - t = self.getchar() - while (t == c): - num += 1 - t = self.getchar() - - finally: - if (c == '['): yield (Tokens.SQRE_OPEN, num) - elif (c == ']'): yield (Tokens.SQRE_CLOSE, num) - elif (c == '{'): yield (Tokens.CURL_OPEN, num) - elif (c == '}'): yield (Tokens.CURL_CLOSE, num) - elif (c == '<'): yield (Tokens.ANGL_OPEN, num) - elif (c == '>'): yield (Tokens.ANGL_CLOSE, num) - elif (c == '='): yield (Tokens.EQUAL_SIGN, num) - elif (c == '\''): yield(Tokens.APOSTROPHE, num) - elif (c == '*'): yield (Tokens.ASTERISK, num) - elif (c == ':'): yield (Tokens.COLON, num) - elif (c == ';'): yield (Tokens.SEMICOLON, num) - elif (c == '#'): yield (Tokens.HASH, num) + + if (c == '['): yield (Tokens.SQRE_OPEN, c) + elif (c == ']'): yield (Tokens.SQRE_CLOSE, c) + elif (c == '}'): yield (Tokens.CURL_CLOSE, c) + elif (c == '<'): yield (Tokens.ANGL_OPEN, c) + elif (c == '>'): yield (Tokens.ANGL_CLOSE, c) + elif (c == '='): yield (Tokens.EQUAL_SIGN, c) + elif (c == '\''): yield(Tokens.APOSTROPHE, c) + elif (c == '*'): yield (Tokens.ASTERISK, c) + elif (c == ':'): yield (Tokens.COLON, c) + elif (c == ';'): yield (Tokens.SEMICOLON, c) + elif (c == '#'): yield (Tokens.HASH, c) + c = self.getchar() + elif (c == '{'): + if text: + yield (Tokens.TEXT, text) + text = '' + t = self.getchar() + if (t == '|'): + yield (Tokens.TAB_OPEN, '{|') + c = self.getchar() + else: + yield (Tokens.CURL_OPEN, '{') + c = t elif (c == '|'): if text: yield (Tokens.TEXT, text) text = '' - try: - t = self.getchar() - except StopIteration: - yield (Tokens.PIPE, None) - raise + t = self.getchar() if (t == '-'): - yield (Tokens.TAB_NEWLINE, None) + yield (Tokens.TAB_NEWLINE, '|-') c = self.getchar() elif (t == '}'): - yield (Tokens.TAB_CLOSE, None) + yield (Tokens.TAB_CLOSE, '|}') c = self.getchar() else: - num = 1 - while (t == c): - num += 1 - t = self.getchar() - yield (Tokens.PIPE, num) - c = t + yield (Tokens.PIPE, None) + c = t elif re.match('\s', c): # whitespace eater pro (TM) if text: yield (Tokens.TEXT, text) @@ -131,4 +132,8 @@ yield (Tokens.EOF, None) def getchar(self): - return self.data.next() \ No newline at end of file + return self.data.next() + +if __name__ == "__main__": + import doctest + doctest.testmod() \ No newline at end of file Modified: trunk/pywikiparser/Parser.py =================================================================== --- trunk/pywikiparser/Parser.py 2007-08-01 00:30:33 UTC (rev 3928) +++ trunk/pywikiparser/Parser.py 2007-08-01 12:31:16 UTC (rev 3929) @@ -28,26 +28,24 @@ data = self.lex.peek() if data[0] in tokens: - return self.lex.next() + return self.lex.next()[1] else: raise ParseError('%r is not one of %r' % (data[0], tokens)) - - def expecttext(self): - data = self.lex.peek() - if data[0] in [Tokens.TEXT, Tokens.WHITESPACE]: - return self.lex.next() - elif data[0] in [Tokens.EQUAL_SIGN, Tokens.APOSTROPHE, Tokens.ASTERISK, - Tokens.COLON, Tokens.SEMICOLON, Tokens.HASH]: - data = self.lex.next() - return (data[0], data[0].__doc__[0]*data[1]) - else: - raise ParseError('%r is not parsable as text data' % (data[0],)) + def eat(self, tokens): + data = '' + try: + while(True): + data += self.expect(tokens) + except ParseError: + return data + def parse(self, breaktoken=[]): self.root = dom.Element('wikipage') self.par = self.root.appendElement('p') self.italic = False self.bold = False + try: while(True): token = self.lex.peek() @@ -61,7 +59,7 @@ except StopIteration: pass return self.root - + def parsetoken(self, token): # The function to call is parser<token> exec("data = self.parse%s()" % token[0].name, globals(), locals()) @@ -81,8 +79,7 @@ return [] def parseAPOSTROPHE(self): - token = self.expect(Tokens.APOSTROPHE) - num = token[1] + num = len(self.eat(Tokens.APOSTROPHE)) #prepare length if (num == 1): @@ -130,51 +127,38 @@ # Functions that return the input directly def parseSQRE_CLOSE(self): - token = self.expect(Tokens.SQRE_CLOSE) - return [']'*token[1]] + return self.expect(Tokens.SQRE_CLOSE) def parsePIPE(self): - token = self.expect(Tokens.PIPE) - return ['|'*token[1]] + return self.expect(Tokens.PIPE) def parseEQUAL_SIGN(self): - token = self.expect(Tokens.EQUAL_SIGN) - return ['='*token[1]] + return self.expect(Tokens.EQUAL_SIGN) def parseCURL_CLOSE(self): - token = self.expect(Tokens.CURL_CLOSE) - return ['}'*token[1]] + return self.expect(Tokens.CURL_CLOSE) def parseANGL_CLOSE(self): - token = self.expect(Tokens.ANGL_CLOSE) - return ['>'*token[1]] + return self.expect(Tokens.ANGL_CLOSE) def parseASTERISK(self): - token = self.expect(Tokens.ASTERISK) - return ['*'*token[1]] + return self.expect(Tokens.ASTERISK) def parseCOLON(self): - token = self.expect(Tokens.COLON) - return [':'*token[1]] + return self.expect(Tokens.COLON) def parseSEMICOLON(self): - token = self.expect(Tokens.SEMICOLON) - return [';'*token[1]] + return self.expect(Tokens.SEMICOLON) def parseHASH(self): - token = self.expect(Tokens.HASH) - return ['#'*token[1]] + return self.expect(Tokens.HASH) def parseTAB_NEWLINE(self): - token = self.expect(Tokens.TAB_NEWLINE) - return ['|-'] + return self.expect(Tokens.TAB_NEWLINE) def parseTAB_CLOSE(self): - token = self.expect(Tokens.TAB_CLOSE) - return ['|}'] - - - + return self.expect(Tokens.TAB_CLOSE) + # True parser callers def parseWHITESPACE(self): @@ -182,11 +166,7 @@ return self.parseTEXT() def parseTEXT(self): - text = '' - while(True): - try: - text += self.expect([Tokens.TEXT, Tokens.WHITESPACE])[1] - except ParseError: break + text = self.eat([Tokens.TEXT, Tokens.WHITESPACE]) if text: return [text] @@ -204,8 +184,7 @@ except ParseError: pass self.lex.undo() - token = self.expect(Tokens.SQRE_OPEN) - return ['['*token[1]] + return self.expect(Tokens.SQRE_OPEN) def parseCURL_OPEN(self): try: @@ -218,8 +197,7 @@ except ParseError: pass self.lex.undo() - token = self.expect(Tokens.CURL_OPEN) - return ['{'*token[1]] + return self.expect(Tokens.CURL_OPEN) def parseANGL_OPEN(self): try: @@ -227,8 +205,7 @@ except ParseError: pass self.lex.undo() - token = self.expect(Tokens.ANGL_OPEN) - return ['<'*token[1]] + return self.expect(Tokens.ANGL_OPEN) def parseTAB_OPEN(self): try: @@ -236,39 +213,25 @@ except ParseError: pass self.lex.undo() - token = self.expect(Tokens.TAB_OPEN) - return ['{|'] + return self.expect(Tokens.TAB_OPEN) titlere = re.compile(r"[^\^\]#<>\[\|\{\}\n]*$") def parseWikilink(self): retval = dom.Element('') - pre = self.expect(Tokens.SQRE_OPEN)[1]-2 + self.expect(Tokens.SQRE_OPEN) + self.expect(Tokens.SQRE_OPEN) - if pre < 0: - raise ParseError("Not enough opening brackets") - elif pre > 0: - retval.append('['*pre) + pre = self.eat(Tokens.SQRE_OPEN) + if pre: + retval.append(pre) - title = '' - while(True): - try: - data = self.expecttext()[1] - print data - except ParseError: break - if not self.titlere.match(data): - raise ParseError("Illegal page title") - else: - title += data + title = self.eat(Tokens.TEXT) # temp. needs to allow templates etc. link = retval.appendElement('wikilink') link.appendElement('url').append(title) - - aft = self.expect(Tokens.SQRE_CLOSE)[1]-2 - if aft < 0: - raise ParseError("Not enough closing brackets") - elif aft > 0: - self.lex.push((Tokens.SQRE_CLOSE, aft)) - + + self.expect(Tokens.SQRE_CLOSE) + self.expect(Tokens.SQRE_CLOSE) return retval

1 0

SVN: [3928] trunk/pywikipedia/weblinkchecker.py
by wikipedian＠svn.wikimedia.org 31 Jul '07

31 Jul '07

Revision: 3928 Author: wikipedian Date: 2007-08-01 00:30:33 +0000 (Wed, 01 Aug 2007) Log Message: ----------- made it possible to use all the typical parameters such as -ref:, -links:, -file:, and -weblink:. added -namespace: parameter. Modified Paths: -------------- trunk/pywikipedia/weblinkchecker.py Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-07-31 16:03:31 UTC (rev 3927) +++ trunk/pywikipedia/weblinkchecker.py 2007-08-01 00:30:33 UTC (rev 3928) @@ -9,7 +9,7 @@ The bot will store all links found dead in a .dat file in the deadlinks subdirectory. To avoid the removing of links which are only temporarily -unavailable, the bot only reports links which were reported dead at least +unavailable, the bot ONLY reports links which were reported dead at least two times, with a time lag of at least one week. Such links will be logged to a .txt file in the deadlinks subdirectory. @@ -29,7 +29,7 @@ python weblinkchecker.py -start:Example_page Loads all wiki pages using the Special:Allpages feature, starting at "Example page" - + python weblinkchecker.py Example page Only checks links found in the wiki page "Example page" """ @@ -91,22 +91,6 @@ re.compile('.*[\./(a)]berlinonline.de(/.*)?'), # a de: user wants to fix them by hand and doesn't want them to be deleted, see [[de:Benutzer:BLueFiSH.as/BZ]]. ] -class Global(object): - talk = config.report_dead_links_on_talk - - def handleArgs(self, args): - unhandledArguments = [] - for arg in args: - if arg == '-talk': - self.talk = True - elif arg == '-notalk': - self.talk = False - else: - unhandledArguments.append(arg) - return unhandledArguments - -globalvar = Global() - def weblinksIn(text, withoutBracketed = False, onlyBracketed = False): text = wikipedia.removeDisabledParts(text) @@ -546,10 +530,9 @@ Robot which will use several LinkCheckThreads at once to search for dead weblinks on pages provided by the given generator. ''' - def __init__(self, generator, start ='!'): + def __init__(self, generator): self.generator = generator - self.start = start - if globalvar.talk: + if config.report_dead_links_on_talk: #wikipedia.output("Starting talk page thread") reportThread = DeadLinkReportThread() # thread dies when program terminates @@ -598,25 +581,37 @@ def main(): gen = None - start = '!' - pageTitle = [] - args = wikipedia.handleArgs() - args = globalvar.handleArgs(args) - - for arg in args: - if arg.startswith('-start:'): - start = arg[7:] + singlePageTitle = [] + # Which namespaces should be processed? + # default to [] which means all namespaces will be processed + namespaces = [] + # This factory is responsible for processing command line arguments + # that are also used by other scripts and that determine on which pages + # to work on. + genFactory = pagegenerators.GeneratorFactory() + + for arg in wikipedia.handleArgs(): + if arg == '-talk': + config.report_dead_links_on_talk = True + elif arg == '-notalk': + config.report_dead_links_on_talk = False + elif arg.startswith('-namespace:'): + namespaces.append(int(arg[11:])) else: - pageTitle.append(arg) + generator = genFactory.handleArg(arg) + if generator: + gen = generator + else: + singlePageTitle.append(arg) - if pageTitle: - pageTitle = ' '.join(pageTitle) - page = wikipedia.Page(wikipedia.getSite(), pageTitle) + if singlePageTitle: + singlePageTitle = ' '.join(singlePageTitle) + page = wikipedia.Page(wikipedia.getSite(), singlePageTitle) gen = iter([page]) - else: - gen = pagegenerators.AllpagesPageGenerator(start) if gen: + if namespaces != []: + gen = pagegenerators.NamespaceFilterPageGenerator(gen, namespaces) gen = pagegenerators.PreloadingGenerator(gen, pageNumber = 240) gen = pagegenerators.RedirectFilterPageGenerator(gen) bot = WeblinkCheckerRobot(gen) @@ -651,7 +646,7 @@ bot.history.save() else: wikipedia.showHelp() - + if __name__ == "__main__": try: main()

1 0

SVN: [3927] trunk/pywikipedia/category.py
by leogregianin＠svn.wikimedia.org 31 Jul '07

31 Jul '07

Revision: 3927 Author: leogregianin Date: 2007-07-31 16:03:31 +0000 (Tue, 31 Jul 2007) Log Message: ----------- Fix bug 1668061: Harmonize category.py's keybindings Modified Paths: -------------- trunk/pywikipedia/category.py Modified: trunk/pywikipedia/category.py =================================================================== --- trunk/pywikipedia/category.py 2007-07-31 12:17:29 UTC (rev 3926) +++ trunk/pywikipedia/category.py 2007-07-31 16:03:31 UTC (rev 3927) @@ -568,7 +568,7 @@ # layout: we don't expect a cat to have more than 100 subcats wikipedia.output(u'%2d - Move down to %s' % (i, subcatlist[i].title())) print ' j - Jump to another category' - print ' n - Skip this article' + print ' s - Skip this article' print ' r - Remove this category tag' print ' ? - Print first part of the page (longer and longer)' wikipedia.output(u'Enter - Save category as %s' % current_cat.title()) @@ -577,7 +577,7 @@ while not flag: print '' choice=wikipedia.input(u'Choice:') - if choice == 'n': + if choice in ['s', 'S']: flag = True elif choice == '': wikipedia.output(u'Saving category as %s' % current_cat.title()) @@ -586,13 +586,13 @@ else: catlib.change_category(article, original_cat, current_cat) flag = True - elif choice == 'j': + elif choice in ['j', 'J']: newCatTitle = wikipedia.input(u'Please enter the category the article should be moved to:') newCat = catlib.Category(wikipedia.getSite(), 'Category:' + newCatTitle) # recurse into chosen category self.move_to_category(article, original_cat, newCat) flag = True - elif choice == 'r': + elif choice in ['r', 'R']: # remove the category tag catlib.change_category(article, original_cat, None) flag = True

1 0

SVN: [3926] trunk/pywikipedia/titletranslate.py
by wikipedian＠svn.wikimedia.org 31 Jul '07

31 Jul '07

Revision: 3926 Author: wikipedian Date: 2007-07-31 12:17:29 +0000 (Tue, 31 Jul 2007) Log Message: ----------- translate namespaces, so that when you run this: python interwiki.py -lang:de "Kategorie:Tour de France" -hint:fr the bot will not load [[fr:Kategorie:Tour de France]], but [[fr:Cat?\195?\169gorie:Tour de France]]. Modified Paths: -------------- trunk/pywikipedia/titletranslate.py Modified: trunk/pywikipedia/titletranslate.py =================================================================== --- trunk/pywikipedia/titletranslate.py 2007-07-29 22:18:58 UTC (rev 3925) +++ trunk/pywikipedia/titletranslate.py 2007-07-31 12:17:29 UTC (rev 3926) @@ -31,7 +31,12 @@ # if given as -hint:xy or -hint:xy:, assume that there should # be a page in language xy with the same title as the page # we're currently working on - newname = page.title() + ns = page.namespace() + if ns: + newname = u'%s:%s' % (site.family.namespace('_default', ns), page.titleWithoutNamespace()) + else: + # article in the main namespace + newname = page.title() try: number = int(codes) codes = site.family.languages_by_size[:number]

1 0

Splitting discussion and svn updates
by Merlijn van Deen 30 Jul '07

30 Jul '07

After moving the mailing list, I recieved two reports of people who would like to split the list into a discussion list and a list that reports SVN updates. When recieving the mails it is, of course, possible to filter client-side at '[Pywikipedia-l] SVN: [', and as the entire list is reasonably low-volume, that should be possible for modem users. This would be my preferred system, but I would like to hear (read) some comments on splitting the list into a development list (which would probably be pywikipedia-l@wikimedia) and a discussion list (which would then be pywikipediabot-users@sourceforge or a new list@wikimedia). Thanks in advance for your responses, valhallasw

2 1

SVN: [3925] trunk/pywikiparser
by valhallasw＠svn.wikimedia.org 29 Jul '07

29 Jul '07

Revision: 3925 Author: valhallasw Date: 2007-07-29 22:18:58 +0000 (Sun, 29 Jul 2007) Log Message: ----------- Late night commit: Parser.py: * parseTEXT now has try/except inside while(True) and uses break instead of pass * Added text parser function (accepts all text except []{}<>|) * Added simple wikilink parser (accepts wikilinks of format [[link]], check wikilink legality) Lexer.py: * Updated ASTERISK description Modified Paths: -------------- trunk/pywikiparser/Lexer.py trunk/pywikiparser/Parser.py Modified: trunk/pywikiparser/Lexer.py =================================================================== --- trunk/pywikiparser/Lexer.py 2007-07-29 21:17:39 UTC (rev 3924) +++ trunk/pywikiparser/Lexer.py 2007-07-29 22:18:58 UTC (rev 3925) @@ -25,7 +25,7 @@ ('PIPE', '| Pipe symbol'), ('EQUAL_SIGN', '= Equal sign'), ('APOSTROPHE', '\' Apostrophe'), - ('ASTERISK', '* Star sign'), + ('ASTERISK', '* Asterisk'), ('COLON', ': Colon'), ('SEMICOLON', '; Semicolon'), ('HASH', '# Hash symbol'), Modified: trunk/pywikiparser/Parser.py =================================================================== --- trunk/pywikiparser/Parser.py 2007-07-29 21:17:39 UTC (rev 3924) +++ trunk/pywikiparser/Parser.py 2007-07-29 22:18:58 UTC (rev 3925) @@ -8,6 +8,7 @@ __version__ = '$Id$' import warnings +import re import ObjectTree as dom from BufferedReader import BufferedReader @@ -30,6 +31,17 @@ return self.lex.next() else: raise ParseError('%r is not one of %r' % (data[0], tokens)) + + def expecttext(self): + data = self.lex.peek() + if data[0] in [Tokens.TEXT, Tokens.WHITESPACE]: + return self.lex.next() + elif data[0] in [Tokens.EQUAL_SIGN, Tokens.APOSTROPHE, Tokens.ASTERISK, + Tokens.COLON, Tokens.SEMICOLON, Tokens.HASH]: + data = self.lex.next() + return (data[0], data[0].__doc__[0]*data[1]) + else: + raise ParseError('%r is not parsable as text data' % (data[0],)) def parse(self, breaktoken=[]): self.root = dom.Element('wikipage') @@ -166,14 +178,16 @@ # True parser callers def parseWHITESPACE(self): + # Todo: return self.parseTEXT() def parseTEXT(self): text = '' - try: - while(True): - text += self.expect([Tokens.TEXT, Tokens.WHITESPACE])[1] - except ParseError: pass + while(True): + try: + text += self.expect([Tokens.TEXT, Tokens.WHITESPACE])[1] + except ParseError: break + if text: return [text] else: @@ -183,7 +197,7 @@ try: return self.parseWikilink() except ParseError: pass - + self.lex.undo() try: return self.parseExternallink() @@ -224,9 +238,39 @@ self.lex.undo() token = self.expect(Tokens.TAB_OPEN) return ['{|'] - + + titlere = re.compile(r"[^\^\]#<>\[\|\{\}\n]*$") def parseWikilink(self): - raise ParseError("Needs implementation") + retval = dom.Element('') + pre = self.expect(Tokens.SQRE_OPEN)[1]-2 + + if pre < 0: + raise ParseError("Not enough opening brackets") + elif pre > 0: + retval.append('['*pre) + + title = '' + while(True): + try: + data = self.expecttext()[1] + print data + except ParseError: break + if not self.titlere.match(data): + raise ParseError("Illegal page title") + else: + title += data + + link = retval.appendElement('wikilink') + link.appendElement('url').append(title) + + aft = self.expect(Tokens.SQRE_CLOSE)[1]-2 + if aft < 0: + raise ParseError("Not enough closing brackets") + elif aft > 0: + self.lex.push((Tokens.SQRE_CLOSE, aft)) + + return retval + def parseExternallink(self): raise ParseError("Needs implementation")

1 0

SVN: [3924] trunk/pywikiparser
by valhallasw＠svn.wikimedia.org 29 Jul '07

29 Jul '07

Revision: 3924 Author: valhallasw Date: 2007-07-29 21:17:39 +0000 (Sun, 29 Jul 2007) Log Message: ----------- STAR -> ASTERISK (Maybe engrish isn't that bad at all ^_^) Modified Paths: -------------- trunk/pywikiparser/Lexer.py trunk/pywikiparser/Parser.py Modified: trunk/pywikiparser/Lexer.py =================================================================== --- trunk/pywikiparser/Lexer.py 2007-07-29 21:09:30 UTC (rev 3923) +++ trunk/pywikiparser/Lexer.py 2007-07-29 21:17:39 UTC (rev 3924) @@ -25,7 +25,7 @@ ('PIPE', '| Pipe symbol'), ('EQUAL_SIGN', '= Equal sign'), ('APOSTROPHE', '\' Apostrophe'), - ('STAR', '* Star sign'), + ('ASTERISK', '* Star sign'), ('COLON', ': Colon'), ('SEMICOLON', '; Semicolon'), ('HASH', '# Hash symbol'), @@ -80,7 +80,7 @@ elif (c == '>'): yield (Tokens.ANGL_CLOSE, num) elif (c == '='): yield (Tokens.EQUAL_SIGN, num) elif (c == '\''): yield(Tokens.APOSTROPHE, num) - elif (c == '*'): yield (Tokens.STAR, num) + elif (c == '*'): yield (Tokens.ASTERISK, num) elif (c == ':'): yield (Tokens.COLON, num) elif (c == ';'): yield (Tokens.SEMICOLON, num) elif (c == '#'): yield (Tokens.HASH, num) Modified: trunk/pywikiparser/Parser.py =================================================================== --- trunk/pywikiparser/Parser.py 2007-07-29 21:09:30 UTC (rev 3923) +++ trunk/pywikiparser/Parser.py 2007-07-29 21:17:39 UTC (rev 3924) @@ -137,8 +137,8 @@ token = self.expect(Tokens.ANGL_CLOSE) return ['>'*token[1]] - def parseSTAR(self): - token = self.expect(Tokens.STAR) + def parseASTERISK(self): + token = self.expect(Tokens.ASTERISK) return ['*'*token[1]] def parseCOLON(self):

1 0

Jump to page:

2025

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

Pywikipedia-l