Revision: 4691
Author: russblau
Date: 2007-12-10 17:29:45 +0000 (Mon, 10 Dec 2007)
Log Message:
-----------
Implement -xmlstart parameter (requested in Patch #1649080)
Modified Paths:
--------------
trunk/pywikipedia/replace.py
Modified: trunk/pywikipedia/replace.py
===================================================================
--- trunk/pywikipedia/replace.py 2007-12-10 16:59:01 UTC (rev 4690)
+++ trunk/pywikipedia/replace.py 2007-12-10 17:29:45 UTC (rev 4691)
@@ -23,6 +23,10 @@
-nocase Use case insensitive regular expressions.
+-xmlstart (Only works with -xml) Skip all articles in the XML dump
+ before the one specified (may also be given as
+ -xmlstart:Article).
+
-excepttitle:XYZ Skip pages with titles that contain XYZ. If the -regex argument
is given, XYZ will be regarded as a regular expression.
@@ -147,6 +151,7 @@
These pages will be retrieved from a local XML dump file.
Arguments:
* xmlFilename - The dump's path, either absolute or relative
+ * xmlStart - Skip all articles in the dump before this one
* replacements - A list of 2-tuples of original text (as a
compiled regular expression) and replacement
text (as a string).
@@ -155,10 +160,12 @@
constructor below.
"""
- def __init__(self, xmlFilename, replacements, exceptions):
+ def __init__(self, xmlFilename, xmlStart, replacements, exceptions):
self.xmlFilename = xmlFilename
self.replacements = replacements
self.exceptions = exceptions
+ self.xmlStart = xmlStart
+ self.skipping = bool(xmlStart)
self.excsInside = []
if self.exceptions.has_key('inside-tags'):
@@ -174,18 +181,32 @@
return self
def next(self):
- while True:
+ try:
+ while True:
+ try:
+ entry = self.parser.next()
+ except StopIteration:
+ raise
+ if self.skipping:
+ if entry.title != self.xmlStart:
+ continue
+ self.skipping = False
+ if not self.isTitleExcepted(entry.title) \
+ and not self.isTextExcepted(entry.text):
+ new_text = entry.text
+ for old, new in self.replacements:
+ new_text = wikipedia.replaceExcept(new_text, old, new, self.excsInside)
+ if new_text != entry.text:
+ return wikipedia.Page(self.site, entry.title)
+ except KeyboardInterrupt:
try:
- entry = self.parser.next()
- except StopIteration:
- raise
- if not self.isTitleExcepted(entry.title) \
- and not self.isTextExcepted(entry.text):
- new_text = entry.text
- for old, new in self.replacements:
- new_text = wikipedia.replaceExcept(new_text, old, new, self.excsInside)
- if new_text != entry.text:
- return wikipedia.Page(self.site, entry.title)
+ if not self.skipping:
+ wikipedia.output(
+ 'To resume, use "-xmlstart:%s" on the command line.'
+ % entry.title)
+ except NameError:
+ pass
+ raise KeyboardInterrupt
def isTitleExcepted(self, title):
if self.exceptions.has_key('title'):
@@ -404,6 +425,12 @@
for arg in wikipedia.handleArgs():
if arg == '-regex':
regex = True
+ elif arg.startswith('-xmlstart'):
+ if len(arg) == 9:
+ xmlStart = wikipedia.input(
+ u'Please enter the dumped article to start with:')
+ else:
+ xmlStart = arg[10:]
elif arg.startswith('-xml'):
if len(arg) == 4:
xmlFilename = wikipedia.input(
@@ -527,7 +554,12 @@
exceptions[exceptionCategory] = patterns
if xmlFilename:
- gen = XmlDumpReplacePageGenerator(xmlFilename, replacements, exceptions)
+ try:
+ xmlStart
+ except NameError:
+ xmlStart = None
+ gen = XmlDumpReplacePageGenerator(xmlFilename, xmlStart,
+ replacements, exceptions)
elif useSql:
whereClause = 'WHERE (%s)' % ' OR '.join(["old_text RLIKE '%s'" % prepareRegexForMySQL(old.pattern) for (old, new) in replacements])
if exceptions:
@@ -553,7 +585,7 @@
wikipedia.stopme()
sys.exit()
if namespaces != []:
- gen = pagegenerators.NamespaceFilterPageGenerator(gen, namespaces)
+ gen = pagegenerators.NamespaceFilterPageGenerator(gen, namespaces)
if xmlFilename:
# XML parsing is slow enough that preloading would make bot even slower
preloadingGen = gen