Revision: 3914
Author: valhallasw
Date: 2007-07-27 15:27:56 +0000 (Fri, 27 Jul 2007)
Log Message:
-----------
moved wikitext parser to own project directory
Added Paths:
-----------
trunk/pywikiparser/
trunk/pywikiparser/parser.py
Removed Paths:
-------------
trunk/pywikipedia/parser.py
Copied: trunk/pywikiparser/parser.py (from rev 3913, trunk/pywikipedia/parser.py)
===================================================================
--- trunk/pywikiparser/parser.py (rev 0)
+++ trunk/pywikiparser/parser.py 2007-07-27 15:27:56 UTC (rev 3914)
@@ -0,0 +1,217 @@
+# -*- coding: utf-8 -*-
+""" Mediawiki wikitext parser """
+#
+# (C) 2007 Merlijn 'valhallasw' van Deen
+#
+# Distributed under the terms of the MIT license.
+#
+__version__ = '$Id$'
+
+#
+
+import re
+import xml.dom.minidom as dom
+
+# System loosely based on 'the dragon book':
+# Compilers, Principles, Techniques and Tools, Aho, Sethi, Ullman, 1st edition, 1986
+
+class ParseError(Exception):
+ """ booh """
+
+class parser:
+ def __init__(self, string):
+ self.wikipwn = [a for a in lexer(string).lexer()]
+ self.counter = 0
+
+ def expect(self, types, values=None):
+ #print 'Expect: %s %s' % (types, values)
+ token = self.wikipwn[self.counter]
+ if (token[0] not in types):
+ if values:
+ raise ParseError("Expected one of (%r, %r), got %r" % (types,
values, token))
+ else:
+ raise ParseError("Expected one of (%r), got %r" % (types,
token))
+ if values:
+ if (token[1] not in values):
+ raise ParseError("Expected one of (%r, %r), got %r" % (types,
values, token))
+ self.counter += 1
+ return token
+
+ def parsetext(self):
+ data = ''
+ try:
+ while(True): data += self.expect([lexer.TEXT, lexer.WHITESPACE])[1]
+ except ParseError, e: pass
+ return data
+
+ def parseurl(self):
+ pre = self.expect([lexer.SQRE_OPEN])[1]-1
+ url = self.expect([lexer.TEXT])[1]
+ # checkurl, raise ParseError
+ try:
+ ws = self.expect([lexer.WHITESPACE])[1]
+ except ParseError:
+ aft = self.expect([lexer.SQRE_CLOSE])[1]-1
+ return ('['*pre, url, ']'*aft)
+
+ if '\n' in ws:
+ raise ParseError('No newlines allowed in external links')
+ desc = ''
+ try:
+ while(True): desc += self.expect([lexer.TEXT, lexer.WHITESPACE])[1]
+ except ParseError, e: pass
+ aft = self.expect([lexer.SQRE_CLOSE])[1]-1
+ return ('['*pre, url, desc, ']'*aft)
+
+ def parsewikilink(self):
+ pre = self.expect([lexer.SQRE_OPEN])[1]-2
+ if (pre < 0): raise ParseError('Not a wiki link')
+ page = ''
+ try:
+ while(True): page += self.expect([lexer.TEXT, lexer.WHITESPACE])[1]
+ except ParseError,e: pass
+ # if not re.match(...): raise ParseError
+ try:
+ aft = self.expect([lexer.SQRE_CLOSE])[1]-2
+ except ParseError, e: pass
+ else:
+ if (aft < 0):
+ raise ParseError('Not a wiki link')
+ return ('['*pre, page, ']'*aft)
+ print 'boom.'
+ return 0
+
+ def parseone(self):
+ token = self.wikipwn[self.counter]
+ if (token[0] == lexer.EOF):
+ raise StopIteration
+ if (token[0] == lexer.TEXT or token[0] == lexer.WHITESPACE): #text
+ try: return self.parsetext();
+ except ParseError, e: pass
+ if (token[0] == lexer.SQRE_OPEN): #wikilink or external link
+ begin = self.counter
+ try: return self.parsewikilink();
+ except ParseError, e: pass
+ self.counter = begin
+ try: return self.parseurl();
+ except ParseError, e: pass
+ self.counter = begin
+ return ('[' * self.expect([lexer.SQRE_OPEN])[1])
+
+ try: self.expect([lexer.SQRE_OPEN], [1])
+ except ParseError, e: pass
+ else: return self.parseurl()
+
+ # Wikilink
+ try: self.expect([lexer.SQRE_OPEN], [2]) #wikilink
+ except ParseError, e: pass
+
+class lexer:
+ class TEXT:
+ """ Text """
+ class SQRE_OPEN:
+ """ Square bracket open """
+ class SQRE_CLOSE:
+ """ Square bracket close """
+ class PIPE:
+ """ Pipe symbol """
+ class CURL_OPEN:
+ """ Curly bracket open """
+ class CURL_CLOSE:
+ """ Curly bracket close """
+ class ANGL_OPEN:
+ """ Angular bracket open """
+ class ANGL_CLOSE:
+ """ Angular bracket close """
+ class NEWPAR:
+ """ New paragraph """
+ class TAB_OPEN:
+ """ Table open """
+ class TAB_NEWLINE:
+ """ Table new row """
+ class TAB_CLOSE:
+ """ Table close """
+ class WHITESPACE:
+ """ Whitespace """
+ class EQUAL_SIGN:
+ """ Equal sign """
+ class APOSTROPHE:
+ """ Apostrophe """
+ class EOF:
+ """ End Of File """
+
+ def __init__(self, string):
+ self.wikipwn = (a for a in string)
+
+ def lexer(self):
+ text = ''
+ try:
+ c = self.getchar()
+ while True:
+ if (c in ('[', ']', '{', '}',
'<', '>', '=', '\'')):
+ if text:
+ yield (lexer.TEXT, text)
+ text = ''
+ num = 1
+ try:
+ t = self.getchar()
+ while (t == c):
+ t = self.getchar()
+ num += 1
+ finally:
+ if (c == '['): yield (lexer.SQRE_OPEN, num)
+ elif (c == ']'): yield (lexer.SQRE_CLOSE, num)
+ elif (c == '{'): yield (lexer.CURL_OPEN, num)
+ elif (c == '}'): yield (lexer.CURL_CLOSE, num)
+ elif (c == '<'): yield (lexer.ANGL_OPEN, num)
+ elif (c == '>'): yield (lexer.ANGL_CLOSE, num)
+ elif (c == '='): yield (lexer.EQUAL_SIGN, num)
+ elif (c == '\''): yield(lexer.APOSTROPHE, num)
+ c = t
+ elif (c == '|'):
+ if text:
+ yield (lexer.TEXT, text)
+ text = ''
+ try:
+ t = self.getchar()
+ except StopIteration:
+ yield (lexer.PIPE, None)
+ raise
+
+ if (t == '-'):
+ yield (lexer.TAB_NEWLINE, None)
+ c = self.getchar()
+ elif (t == '}'):
+ yield (lexer.TAB_CLOSE, None)
+ c = self.getchar()
+ else:
+ yield (lexer.PIPE, None)
+ c = t
+ elif re.match('\s', c): # whitespace eater pro (TM)
+ if text:
+ yield (lexer.TEXT, text)
+ text = ''
+ ws = ''
+ try:
+ while re.match('\s', c):
+ ws += c
+ c = self.getchar() #eat up remaining whitespace
+ finally:
+ if (ws.count('\n') > 1):
+ yield (lexer.NEWPAR, ws)
+ else:
+ yield (lexer.WHITESPACE, ws)
+ else:
+ text = text + c
+ c = self.getchar()
+ except StopIteration: pass
+ if text:
+ yield (lexer.TEXT, text)
+ yield (lexer.EOF, None)
+
+ def getchar(self):
+ return self.wikipwn.next()
+
+
+
+
\ No newline at end of file
Deleted: trunk/pywikipedia/parser.py
===================================================================
--- trunk/pywikipedia/parser.py 2007-07-27 14:26:25 UTC (rev 3913)
+++ trunk/pywikipedia/parser.py 2007-07-27 15:27:56 UTC (rev 3914)
@@ -1,217 +0,0 @@
-# -*- coding: utf-8 -*-
-""" Mediawiki wikitext parser """
-#
-# (C) 2007 Merlijn 'valhallasw' van Deen
-#
-# Distributed under the terms of the MIT license.
-#
-__version__ = '$Id$'
-
-#
-
-import re
-import xml.dom.minidom as dom
-
-# System loosely based on 'the dragon book':
-# Compilers, Principles, Techniques and Tools, Aho, Sethi, Ullman, 1st edition, 1986
-
-class ParseError(Exception):
- """ booh """
-
-class parser:
- def __init__(self, string):
- self.wikipwn = [a for a in lexer(string).lexer()]
- self.counter = 0
-
- def expect(self, types, values=None):
- #print 'Expect: %s %s' % (types, values)
- token = self.wikipwn[self.counter]
- if (token[0] not in types):
- if values:
- raise ParseError("Expected one of (%r, %r), got %r" % (types,
values, token))
- else:
- raise ParseError("Expected one of (%r), got %r" % (types,
token))
- if values:
- if (token[1] not in values):
- raise ParseError("Expected one of (%r, %r), got %r" % (types,
values, token))
- self.counter += 1
- return token
-
- def parsetext(self):
- data = ''
- try:
- while(True): data += self.expect([lexer.TEXT, lexer.WHITESPACE])[1]
- except ParseError, e: pass
- return data
-
- def parseurl(self):
- pre = self.expect([lexer.SQRE_OPEN])[1]-1
- url = self.expect([lexer.TEXT])[1]
- # checkurl, raise ParseError
- try:
- ws = self.expect([lexer.WHITESPACE])[1]
- except ParseError:
- aft = self.expect([lexer.SQRE_CLOSE])[1]-1
- return ('['*pre, url, ']'*aft)
-
- if '\n' in ws:
- raise ParseError('No newlines allowed in external links')
- desc = ''
- try:
- while(True): desc += self.expect([lexer.TEXT, lexer.WHITESPACE])[1]
- except ParseError, e: pass
- aft = self.expect([lexer.SQRE_CLOSE])[1]-1
- return ('['*pre, url, desc, ']'*aft)
-
- def parsewikilink(self):
- pre = self.expect([lexer.SQRE_OPEN])[1]-2
- if (pre < 0): raise ParseError('Not a wiki link')
- page = ''
- try:
- while(True): page += self.expect([lexer.TEXT, lexer.WHITESPACE])[1]
- except ParseError,e: pass
- # if not re.match(...): raise ParseError
- try:
- aft = self.expect([lexer.SQRE_CLOSE])[1]-2
- except ParseError, e: pass
- else:
- if (aft < 0):
- raise ParseError('Not a wiki link')
- return ('['*pre, page, ']'*aft)
- print 'boom.'
- return 0
-
- def parseone(self):
- token = self.wikipwn[self.counter]
- if (token[0] == lexer.EOF):
- raise StopIteration
- if (token[0] == lexer.TEXT or token[0] == lexer.WHITESPACE): #text
- try: return self.parsetext();
- except ParseError, e: pass
- if (token[0] == lexer.SQRE_OPEN): #wikilink or external link
- begin = self.counter
- try: return self.parsewikilink();
- except ParseError, e: pass
- self.counter = begin
- try: return self.parseurl();
- except ParseError, e: pass
- self.counter = begin
- return ('[' * self.expect([lexer.SQRE_OPEN])[1])
-
- try: self.expect([lexer.SQRE_OPEN], [1])
- except ParseError, e: pass
- else: return self.parseurl()
-
- # Wikilink
- try: self.expect([lexer.SQRE_OPEN], [2]) #wikilink
- except ParseError, e: pass
-
-class lexer:
- class TEXT:
- """ Text """
- class SQRE_OPEN:
- """ Square bracket open """
- class SQRE_CLOSE:
- """ Square bracket close """
- class PIPE:
- """ Pipe symbol """
- class CURL_OPEN:
- """ Curly bracket open """
- class CURL_CLOSE:
- """ Curly bracket close """
- class ANGL_OPEN:
- """ Angular bracket open """
- class ANGL_CLOSE:
- """ Angular bracket close """
- class NEWPAR:
- """ New paragraph """
- class TAB_OPEN:
- """ Table open """
- class TAB_NEWLINE:
- """ Table new row """
- class TAB_CLOSE:
- """ Table close """
- class WHITESPACE:
- """ Whitespace """
- class EQUAL_SIGN:
- """ Equal sign """
- class APOSTROPHE:
- """ Apostrophe """
- class EOF:
- """ End Of File """
-
- def __init__(self, string):
- self.wikipwn = (a for a in string)
-
- def lexer(self):
- text = ''
- try:
- c = self.getchar()
- while True:
- if (c in ('[', ']', '{', '}',
'<', '>', '=', '\'')):
- if text:
- yield (lexer.TEXT, text)
- text = ''
- num = 1
- try:
- t = self.getchar()
- while (t == c):
- t = self.getchar()
- num += 1
- finally:
- if (c == '['): yield (lexer.SQRE_OPEN, num)
- elif (c == ']'): yield (lexer.SQRE_CLOSE, num)
- elif (c == '{'): yield (lexer.CURL_OPEN, num)
- elif (c == '}'): yield (lexer.CURL_CLOSE, num)
- elif (c == '<'): yield (lexer.ANGL_OPEN, num)
- elif (c == '>'): yield (lexer.ANGL_CLOSE, num)
- elif (c == '='): yield (lexer.EQUAL_SIGN, num)
- elif (c == '\''): yield(lexer.APOSTROPHE, num)
- c = t
- elif (c == '|'):
- if text:
- yield (lexer.TEXT, text)
- text = ''
- try:
- t = self.getchar()
- except StopIteration:
- yield (lexer.PIPE, None)
- raise
-
- if (t == '-'):
- yield (lexer.TAB_NEWLINE, None)
- c = self.getchar()
- elif (t == '}'):
- yield (lexer.TAB_CLOSE, None)
- c = self.getchar()
- else:
- yield (lexer.PIPE, None)
- c = t
- elif re.match('\s', c): # whitespace eater pro (TM)
- if text:
- yield (lexer.TEXT, text)
- text = ''
- ws = ''
- try:
- while re.match('\s', c):
- ws += c
- c = self.getchar() #eat up remaining whitespace
- finally:
- if (ws.count('\n') > 1):
- yield (lexer.NEWPAR, ws)
- else:
- yield (lexer.WHITESPACE, ws)
- else:
- text = text + c
- c = self.getchar()
- except StopIteration: pass
- if text:
- yield (lexer.TEXT, text)
- yield (lexer.EOF, None)
-
- def getchar(self):
- return self.wikipwn.next()
-
-
-
-
\ No newline at end of file