Pywikipedia-svn April 2009

pywikipedia-svn@lists.wikimedia.org

8 participants
162 discussions

SVN: [6770] trunk/pywikipedia/commonsdelinker/delinker.py

by nicdumz＠svn.wikimedia.org

Revision: 6770 Author: nicdumz Date: 2009-04-30 09:21:06 +0000 (Thu, 30 Apr 2009) Log Message: ----------- [ 2783898 ] Fix links in CommonsDelinker edit summary on Commons Kudos to Ilmari Karonen for this patch :) Modified Paths: -------------- trunk/pywikipedia/commonsdelinker/delinker.py Modified: trunk/pywikipedia/commonsdelinker/delinker.py =================================================================== --- trunk/pywikipedia/commonsdelinker/delinker.py 2009-04-30 09:07:23 UTC (rev 6769) +++ trunk/pywikipedia/commonsdelinker/delinker.py 2009-04-30 09:21:06 UTC (rev 6770) @@ -343,8 +343,9 @@ def get_summary(self, site, image, admin, reason, replacement): """ Get the summary template and substitute the correct values.""" - # FIXME: Don't insert commons: on local delink # FIXME: Hardcode is EVIL + if site.lang != 'commons': + reason = reason.replace('[[', '[[commons:') if replacement: tlp = self.CommonsDelinker.SummaryCache.get(site, 'replace-I18n') else: @@ -354,10 +355,10 @@ if replacement: tlp = tlp.replace('$2', replacement) tlp = tlp.replace('$3', unicode(admin)) - tlp = tlp.replace('$4', unicode(reason).replace('[[', '[[commons:')) + tlp = tlp.replace('$4', unicode(reason)) else: tlp = tlp.replace('$2', unicode(admin)) - tlp = tlp.replace('$3', reason.replace('[[', '[[commons:')) + tlp = tlp.replace('$3', unicode(reason)) return tlp

15 years

SVN: [6769] trunk/pywikipedia/cosmetic_changes.py

by nicdumz＠svn.wikimedia.org

Revision: 6769 Author: nicdumz Date: 2009-04-30 09:07:23 +0000 (Thu, 30 Apr 2009) Log Message: ----------- [ 2783415 ] war: translations for cosmetic_changes Thanks Xqt :) Modified Paths: -------------- trunk/pywikipedia/cosmetic_changes.py Modified: trunk/pywikipedia/cosmetic_changes.py =================================================================== --- trunk/pywikipedia/cosmetic_changes.py 2009-04-30 09:05:12 UTC (rev 6768) +++ trunk/pywikipedia/cosmetic_changes.py 2009-04-30 09:07:23 UTC (rev 6769) @@ -57,6 +57,7 @@ 'pl': u'Robot dokonuje poprawek kosmetycznych', 'pt': u'Bot: Mudanças triviais', 'sv': u'Bot: Kosmetiska ändringar', + 'war': u'Robot: Kosmetiko nga mga pagbag-o', 'zh': u'機器人: 細部更改', } @@ -79,6 +80,7 @@ 'pl': u'; zmiany kosmetyczne', 'pt': u'; mudanças triviais', 'sv': u'kosmetiska ändringar', + 'war': u'; kosmetiko nga mga pagbag-o', 'zh': u'; 細部更改', }

15 years

SVN: [6768] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6768 Author: nicdumz Date: 2009-04-30 09:05:12 +0000 (Thu, 30 Apr 2009) Log Message: ----------- [ 2783410 ] war: translations for interwiki Thanks Xqt :) Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-30 09:00:50 UTC (rev 6767) +++ trunk/pywikipedia/interwiki.py 2009-04-30 09:05:12 UTC (rev 6768) @@ -462,6 +462,7 @@ 'uz': (u'Bot', u'Qoʻshdi', u'Tuzatdi', u'Oʻchirdi'), 'vi': (u'robot ', u'Thêm', u'Dời', u'Thay'), 'vo': (u'bot ', u'läükon', u'moükon', u'votükon'), + 'war':(u'robot ', u'Gindugngan', u'Gintanggal', u'Ginliwat'), 'yi': (u'באט ', u'צוגעלייגט', u'אראפגענומען', u'געענדערט'), 'yue': (u'機械人 ', u'加', u'減', u'改'), 'zh': (u'機器人 ', u'新增', u'移除', u'修改'),

15 years

SVN: [6767] trunk/pywikipedia/wikipedia.py

by nicdumz＠svn.wikimedia.org

Revision: 6767 Author: nicdumz Date: 2009-04-30 09:00:50 +0000 (Thu, 30 Apr 2009) Log Message: ----------- [ 2771272 ] 44 Error Dump Files : print a message on site error, sleep and retry. Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-04-30 08:54:39 UTC (rev 6766) +++ trunk/pywikipedia/wikipedia.py 2009-04-30 09:00:50 UTC (rev 6767) @@ -2974,6 +2974,13 @@ else: if "<title>Wiki does not exist</title>" in data: raise NoSuchSite(u'Wiki %s does not exist yet' % self.site) + elif "</mediawiki>" not in data: + # HTML error Page got thrown because of an internal + # error when fetching a revision. + output(u'Remote site has a problem, it probably ' \ + 'exited our query with an internal Error. ' \ + 'Sleeping for %d seconds...' % self.sleeptime) + self.sleep() elif "<siteinfo>" not in data: # This probably means we got a 'temporary unaivalable' output(u'Got incorrect export page. ' \ 'Sleeping for %d seconds...' % self.sleeptime)

15 years

SVN: [6766] trunk/pywikipedia/wikipedia.py

by nicdumz＠svn.wikimedia.org

Revision: 6766 Author: nicdumz Date: 2009-04-30 08:54:39 +0000 (Thu, 30 Apr 2009) Log Message: ----------- Refactoring sleep bits into GetAll.sleep() Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-04-30 08:46:35 UTC (rev 6765) +++ trunk/pywikipedia/wikipedia.py 2009-04-30 08:54:39 UTC (rev 6766) @@ -2945,6 +2945,7 @@ self.pages = [] self.throttle = throttle self.force = force + self.sleeptime = 15 for page in pages: if (not hasattr(page, '_contents') and not hasattr(page, '_getexception')) or force: @@ -2952,8 +2953,14 @@ elif verbose: output(u"BUGWARNING: %s already done!" % page.aslink()) + def sleep(self): + time.sleep(self.sleeptime) + if self.sleeptime <= 60: + self.sleeptime += 15 + elif self.sleeptime < 360: + self.sleeptime += 60 + def run(self): - dt=15 if self.pages: while True: try: @@ -2961,22 +2968,16 @@ except (socket.error, httplib.BadStatusLine, ServerError): # Print the traceback of the caught exception output(u''.join(traceback.format_exception(*sys.exc_info()))) - output(u'DBG> got network error in _GetAll.run. Sleeping for %d seconds...' % dt) - time.sleep(dt) - if dt <= 60: - dt += 15 - elif dt < 360: - dt += 60 + output(u'DBG> got network error in _GetAll.run. ' \ + 'Sleeping for %d seconds...' % self.sleeptime) + self.sleep() else: if "<title>Wiki does not exist</title>" in data: raise NoSuchSite(u'Wiki %s does not exist yet' % self.site) elif "<siteinfo>" not in data: # This probably means we got a 'temporary unaivalable' - output(u'Got incorrect export page. Sleeping for %d seconds...' % dt) - time.sleep(dt) - if dt <= 60: - dt += 15 - elif dt < 360: - dt += 60 + output(u'Got incorrect export page. ' \ + 'Sleeping for %d seconds...' % self.sleeptime) + self.sleep() else: break R = re.compile(r"\s*<\?xml([^>]*)\?>(.*)",re.DOTALL)

15 years

SVN: [6765] trunk/pywikipedia

by nicdumz＠svn.wikimedia.org

Revision: 6765 Author: nicdumz Date: 2009-04-30 08:46:35 +0000 (Thu, 30 Apr 2009) Log Message: ----------- [ 2782305 ] xmlreader.xmldump allrevision missing title Applying patch by Johan Euphrosine, and adding a few more assertions testing xmlreader Modified Paths: -------------- trunk/pywikipedia/tests/test-xmlreader.py trunk/pywikipedia/xmlreader.py Modified: trunk/pywikipedia/tests/test-xmlreader.py =================================================================== --- trunk/pywikipedia/tests/test-xmlreader.py 2009-04-30 08:28:31 UTC (rev 6764) +++ trunk/pywikipedia/tests/test-xmlreader.py 2009-04-30 08:46:35 UTC (rev 6765) @@ -11,12 +11,19 @@ def test_XmlDumpAllRevs(self): pages = [r for r in xmlreader.XmlDump("data/article-pear.xml", allrevisions=True).parse()] self.assertEquals(4, len(pages)) - self.assertNotEquals("", pages[0].comment) + self.assertEquals(u"Automated conversion", pages[0].comment) + self.assertEquals(u"Pear", pages[0].title) + self.assertEquals(u"24278", pages[0].id) + self.assertTrue(pages[0].text.startswith('Pears are [[tree]]s of')) + self.assertEquals(u"Quercusrobur", pages[1].username) def test_XmlDumpFirstRev(self): pages = [r for r in xmlreader.XmlDump("data/article-pear.xml").parse()] self.assertEquals(1, len(pages)) - self.assertNotEquals("", pages[0].comment) + self.assertEquals(u"Automated conversion", pages[0].comment) + self.assertEquals(u"Pear", pages[0].title) + self.assertEquals(u"24278", pages[0].id) + self.assertTrue(pages[0].text.startswith('Pears are [[tree]]s of')) def test_MediaWikiXmlHandler(self): handler = xmlreader.MediaWikiXmlHandler() Modified: trunk/pywikipedia/xmlreader.py =================================================================== --- trunk/pywikipedia/xmlreader.py 2009-04-30 08:28:31 UTC (rev 6764) +++ trunk/pywikipedia/xmlreader.py 2009-04-30 08:46:35 UTC (rev 6765) @@ -304,7 +304,7 @@ def _parse_all(self, event, elem): """Parser that yields all revisions""" - if event == "start" and elem.tag == "{%s}revision" % self.uri: + if event == "start" and elem.tag == "{%s}page" % self.uri: self._headers(elem) if event == "end" and elem.tag == "{%s}revision" % self.uri:

15 years

SVN: [6764] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6764 Author: nicdumz Date: 2009-04-30 08:28:31 +0000 (Thu, 30 Apr 2009) Log Message: ----------- Not all pages have _contents attribute Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-30 08:24:07 UTC (rev 6763) +++ trunk/pywikipedia/interwiki.py 2009-04-30 08:28:31 UTC (rev 6764) @@ -1432,7 +1432,8 @@ # foundIn can contain either Page or StoredPage objects # calling the destructor on _contents will delete the # disk records if necessary - del page._contents + if hasattr(page, '_contents'): + del page._contents def replaceLinks(self, page, newPages, bot): """

15 years

SVN: [6763] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6763 Author: nicdumz Date: 2009-04-30 08:24:07 +0000 (Thu, 30 Apr 2009) Log Message: ----------- Convert to StoredPage only if the Page will eventually be loaded (aka, if the Page is added to Subject.todo) Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-30 08:13:06 UTC (rev 6762) +++ trunk/pywikipedia/interwiki.py 2009-04-30 08:24:07 UTC (rev 6763) @@ -860,14 +860,14 @@ self.makeForcedStop(counter) return False - if globalvar.contentsondisk: - page = StoredPage(page) if page in self.foundIn: # not new self.foundIn[page].append(linkingPage) return False else: + if globalvar.contentsondisk: + page = StoredPage(page) self.foundIn[page] = [linkingPage] self.todo.add(page) counter.plus(page.site()) @@ -899,9 +899,6 @@ if globalvar.autonomous: wikipedia.output(u"NOTE: Ignoring link from page %s in namespace %i to page %s in namespace %i." % (self.originPage.aslink(True), self.originPage.namespace(), linkedPage.aslink(True), linkedPage.namespace())) # Fill up foundIn, so that we will not write this notice - if globalvar.contentsondisk: - linkedPage = StoredPage(linkedPage) - self.foundIn[linkedPage] = [linkingPage] return True else: @@ -913,9 +910,6 @@ choice = wikipedia.inputChoice('WARNING: %s is in namespace %i, but %s is in namespace %i. Follow it anyway?' % (self.originPage.aslink(True), self.originPage.namespace(), linkedPage.aslink(True), linkedPage.namespace()), ['Yes', 'No', 'Add an alternative', 'give up'], ['y', 'n', 'a', 'g']) if choice != 'y': # Fill up foundIn, so that we will not ask again - if globalvar.contentsondisk: - linkedPage = StoredPage(linkedPage) - self.foundIn[linkedPage] = [linkingPage] if choice == 'g': self.makeForcedStop(counter) @@ -1081,6 +1075,8 @@ wikipedia.output(u"NOTE: %s is redirect to %s" % (page.aslink(True), redirectTargetPage.aslink(True))) if page == self.originPage: if globalvar.initialredirect: + if globalvar.contentsondisk: + redirectTargetPage = StoredPage(redirectTargetPage) self.originPage = redirectTargetPage self.todo.add(redirectTargetPage) counter.plus(redirectTargetPage.site) @@ -1432,8 +1428,11 @@ whole storage file will be eventually removed. """ if globalvar.contentsondisk: - for storedPage in self.foundIn: - storedPage.SPdelContents() + for page in self.foundIn: + # foundIn can contain either Page or StoredPage objects + # calling the destructor on _contents will delete the + # disk records if necessary + del page._contents def replaceLinks(self, page, newPages, bot): """

15 years

SVN: [6762] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6762 Author: nicdumz Date: 2009-04-30 08:13:06 +0000 (Thu, 30 Apr 2009) Log Message: ----------- Converting to StoredPage everywhere Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-30 08:08:00 UTC (rev 6761) +++ trunk/pywikipedia/interwiki.py 2009-04-30 08:13:06 UTC (rev 6762) @@ -796,6 +796,8 @@ pages = titletranslate.translate(self.originPage, hints = hints, auto = globalvar.auto, removebrackets = globalvar.hintnobracket) for page in pages: + if globalvar.contentsondisk: + page = StoredPage(page) self.todo.add(page) self.foundIn[page] = [None] if keephintedsites: @@ -897,6 +899,9 @@ if globalvar.autonomous: wikipedia.output(u"NOTE: Ignoring link from page %s in namespace %i to page %s in namespace %i." % (self.originPage.aslink(True), self.originPage.namespace(), linkedPage.aslink(True), linkedPage.namespace())) # Fill up foundIn, so that we will not write this notice + if globalvar.contentsondisk: + linkedPage = StoredPage(linkedPage) + self.foundIn[linkedPage] = [linkingPage] return True else: @@ -908,6 +913,9 @@ choice = wikipedia.inputChoice('WARNING: %s is in namespace %i, but %s is in namespace %i. Follow it anyway?' % (self.originPage.aslink(True), self.originPage.namespace(), linkedPage.aslink(True), linkedPage.namespace()), ['Yes', 'No', 'Add an alternative', 'give up'], ['y', 'n', 'a', 'g']) if choice != 'y': # Fill up foundIn, so that we will not ask again + if globalvar.contentsondisk: + linkedPage = StoredPage(linkedPage) + self.foundIn[linkedPage] = [linkingPage] if choice == 'g': self.makeForcedStop(counter)

15 years

SVN: [6761] trunk/pywikipedia/interwiki.py

by nicdumz＠svn.wikimedia.org

Revision: 6761 Author: nicdumz Date: 2009-04-30 08:08:00 +0000 (Thu, 30 Apr 2009) Log Message: ----------- Move the cleanup bits in a Subject.clean() function, and make sure it is called after finish(), even if finish() returns early. Modified Paths: -------------- trunk/pywikipedia/interwiki.py Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-04-30 04:52:44 UTC (rev 6760) +++ trunk/pywikipedia/interwiki.py 2009-04-30 08:08:00 UTC (rev 6761) @@ -1410,6 +1410,7 @@ if config.interwiki_backlink: self.reportBacklinks(new, updatedSites) + def clean(self): """ Delete the contents that are stored on disk for this Subject. @@ -1822,6 +1823,7 @@ subj = self.subjects[i] if subj.isDone(): subj.finish(self) + subj.clean() del self.subjects[i] def isDone(self):

15 years

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn April 2009