Pywikipedia-svn

pywikipedia-svn@lists.wikimedia.org

5163 discussions

SVN: [6821] branches/rewrite/pywikibot/site.py
by russblau＠svn.wikimedia.org 04 May '09

04 May '09

Revision: 6821 Author: russblau Date: 2009-05-04 20:43:01 +0000 (Mon, 04 May 2009) Log Message: ----------- Fix bug 2784162 - pagebacklinks(followRedirects=True) doesn't yield redirects Modified Paths: -------------- branches/rewrite/pywikibot/site.py Modified: branches/rewrite/pywikibot/site.py =================================================================== --- branches/rewrite/pywikibot/site.py 2009-05-04 16:33:49 UTC (rev 6820) +++ branches/rewrite/pywikibot/site.py 2009-05-04 20:43:01 UTC (rev 6821) @@ -1011,7 +1011,7 @@ return page._redirtarget def preloadpages(self, pagelist, groupsize=50, templates=False, - langlinks=False): + langlinks=False): """Return a generator to a list of preloaded pages. Note that [at least in current implementation] pages may be iterated @@ -1114,17 +1114,16 @@ # links identified by MediaWiki as redirects may not really be, # so we have to check each "redirect" page and see if it # really redirects to this page - blgen.request["gblfilterredir"] = "nonredirects" redirgen = api.PageGenerator("backlinks", gbltitle=bltitle, site=self, gblfilterredir="redirects") - if "gblnamespace" in blgen.request: - redirgen.request["gblnamespace"] = blgen.request["gblnamespace"] genlist = [blgen] for redir in redirgen: if redir.getRedirectTarget() == page: genlist.append( self.pagebacklinks( - redir, True, None, namespaces)) + redir, followRedirects=True, + filterRedirects=filterRedirects, + namespaces=namespaces)) import itertools return itertools.chain(*genlist) return blgen

1 0

SVN: [6820] trunk/pywikipedia/wikipedia.py
by nicdumz＠svn.wikimedia.org 04 May '09

04 May '09

Revision: 6820 Author: nicdumz Date: 2009-05-04 16:33:49 +0000 (Mon, 04 May 2009) Log Message: ----------- Updating message when XML is incomplete, per http://lists.wikimedia.org/pipermail/pywikipedia-l/2009-May/005845.html Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-05-04 15:54:19 UTC (rev 6819) +++ trunk/pywikipedia/wikipedia.py 2009-05-04 16:33:49 UTC (rev 6820) @@ -2977,8 +2977,7 @@ elif "</mediawiki>" not in data[-20:]: # HTML error Page got thrown because of an internal # error when fetching a revision. - output(u'Remote site has a problem, it probably ' \ - 'exited our query with an internal Error. ' \ + output(u'Received incomplete XML data. ' \ 'Sleeping for %d seconds...' % self.sleeptime) self.sleep() elif "<siteinfo>" not in data: # This probably means we got a 'temporary unaivalable'

1 0

SVN: [6819]
by nicdumz＠svn.wikimedia.org 04 May '09

04 May '09

Revision: 6819 Author: nicdumz Date: 2009-05-04 15:54:19 +0000 (Mon, 04 May 2009) Log Message: ----------- ReplaceLanguageLinks : 'match'.split(str) should be str.split('match') Modified Paths: -------------- branches/rewrite/pywikibot/textlib.py trunk/pywikipedia/wikipedia.py Modified: branches/rewrite/pywikibot/textlib.py =================================================================== --- branches/rewrite/pywikibot/textlib.py 2009-05-04 13:43:47 UTC (rev 6818) +++ branches/rewrite/pywikibot/textlib.py 2009-05-04 15:54:19 UTC (rev 6819) @@ -397,7 +397,7 @@ else: if template: # Do we have a noinclude at the end of the template? - parts = '</noinclude>'.split(s2) + parts = s2.split('</noinclude>') lastpart = parts[-1] if re.match('\s*%s' % marker, lastpart): # Put the langlinks back into the noinclude's Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-05-04 13:43:47 UTC (rev 6818) +++ trunk/pywikipedia/wikipedia.py 2009-05-04 15:54:19 UTC (rev 6819) @@ -3700,7 +3700,7 @@ else: if template: # Do we have a noinclude at the end of the template? - parts = '</noinclude>'.split(s2) + parts = s2.split('</noinclude>') lastpart = parts[-1] if re.match('\s*%s' % marker, lastpart): # Put the langlinks back into the noinclude's

1 0

SVN: [6818] branches/rewrite/pywikibot/page.py
by russblau＠svn.wikimedia.org 04 May '09

04 May '09

Revision: 6818 Author: russblau Date: 2009-05-04 13:43:47 +0000 (Mon, 04 May 2009) Log Message: ----------- Bug in r6807: str.title() capitalizes all words, not just the first one, which is undesirable Modified Paths: -------------- branches/rewrite/pywikibot/page.py Modified: branches/rewrite/pywikibot/page.py =================================================================== --- branches/rewrite/pywikibot/page.py 2009-05-04 01:41:41 UTC (rev 6817) +++ branches/rewrite/pywikibot/page.py 2009-05-04 13:43:47 UTC (rev 6818) @@ -130,7 +130,7 @@ self._revisions = {} # Always capitalize the first letter - self._title = self._title.title() + self._title = self._title[:1].upper() + self._title[1:] def site(self): """Return the Site object for the wiki on which this Page resides."""

1 0

SVN: [6817]
by nicdumz＠svn.wikimedia.org 04 May '09

04 May '09

Revision: 6817 Author: nicdumz Date: 2009-05-04 01:41:41 +0000 (Mon, 04 May 2009) Log Message: ----------- Interwikis in template namespace: applying patch #2780955, first submitted by Tieump Pushing those changes in the rewrite branch too. Modified Paths: -------------- branches/rewrite/pywikibot/textlib.py branches/rewrite/scripts/interwiki.py trunk/pywikipedia/interwiki.py trunk/pywikipedia/wikipedia.py Modified: branches/rewrite/pywikibot/textlib.py =================================================================== --- branches/rewrite/pywikibot/textlib.py 2009-05-04 01:26:35 UTC (rev 6816) +++ branches/rewrite/pywikibot/textlib.py 2009-05-04 01:41:41 UTC (rev 6817) @@ -347,7 +347,8 @@ return removeLanguageLinks(text, site, marker) -def replaceLanguageLinks(oldtext, new, site = None, addOnly = False): +def replaceLanguageLinks(oldtext, new, site = None, addOnly = False, + template = False): """Replace interlanguage links in the text with a new set of links. 'new' should be a dict with the Site objects as keys, and Page or Link @@ -394,7 +395,19 @@ newtext = replaceCategoryLinks(s2, cats, site=site, addOnly=True) else: - newtext = s2.replace(marker,'').strip() + separator + s + if template: + # Do we have a noinclude at the end of the template? + parts = '</noinclude>'.split(s2) + lastpart = parts[-1] + if re.match('\s*%s' % marker, lastpart): + # Put the langlinks back into the noinclude's + regexp = re.compile('</noinclude>\s*%s' % marker) + newtext = regexp.sub(s + '</noinclude>', s2) + else: + # Put the langlinks at the end, inside noinclude's + newtext = s2.replace(marker,'').strip() + separator + u'<noinclude>\n%s</noinclude>\n' % s + else: + newtext = s2.replace(marker,'').strip() + separator + s else: newtext = s2.replace(marker,'') return newtext Modified: branches/rewrite/scripts/interwiki.py =================================================================== --- branches/rewrite/scripts/interwiki.py 2009-05-04 01:26:35 UTC (rev 6816) +++ branches/rewrite/scripts/interwiki.py 2009-05-04 01:41:41 UTC (rev 6817) @@ -1527,7 +1527,9 @@ pywikibot.output(u"Changes to be made: %s" % mods) oldtext = page.get() - newtext = pywikibot.replaceLanguageLinks(oldtext, new, site = page.site()) + newtext = pywikibot.replaceLanguageLinks(oldtext, new, + site = page.site(), + template = (page.namespace() == 10)) if newtext == oldtext: return False if globalvar.debug: Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2009-05-04 01:26:35 UTC (rev 6816) +++ trunk/pywikipedia/interwiki.py 2009-05-04 01:41:41 UTC (rev 6817) @@ -1524,7 +1524,7 @@ wikipedia.output(u"Changes to be made: %s" % mods) oldtext = page.get() - newtext = wikipedia.replaceLanguageLinks(oldtext, new, site = page.site()) + newtext = wikipedia.replaceLanguageLinks(oldtext, new, site = page.site(), template = (page.namespace() == 10) ) if newtext == oldtext: return False if globalvar.debug: Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2009-05-04 01:26:35 UTC (rev 6816) +++ trunk/pywikipedia/wikipedia.py 2009-05-04 01:41:41 UTC (rev 6817) @@ -3658,7 +3658,7 @@ else: return removeLanguageLinks(text, site, marker) -def replaceLanguageLinks(oldtext, new, site = None, addOnly = False): +def replaceLanguageLinks(oldtext, new, site = None, addOnly = False, template = False): """Replace interlanguage links in the text with a new set of links. 'new' should be a dict with the Site objects as keys, and Page objects @@ -3698,7 +3698,19 @@ s2 = removeCategoryLinksAndSeparator(s2.replace(marker,'',cseparatorstripped).strip(), site) + separator + s newtext = replaceCategoryLinks(s2, cats, site=site, addOnly=True) else: - newtext = s2.replace(marker,'').strip() + separator + s + if template: + # Do we have a noinclude at the end of the template? + parts = '</noinclude>'.split(s2) + lastpart = parts[-1] + if re.match('\s*%s' % marker, lastpart): + # Put the langlinks back into the noinclude's + regexp = re.compile('</noinclude>\s*%s' % marker) + newtext = regexp.sub(s + '</noinclude>', s2) + else: + # Put the langlinks at the end, inside noinclude's + newtext = s2.replace(marker,'').strip() + separator + u'<noinclude>\n%s</noinclude>\n' % s + else: + newtext = s2.replace(marker,'').strip() + separator + s else: newtext = s2.replace(marker,'') return newtext

1 0

SVN: [6816] branches/rewrite/scripts/interwiki.py
by nicdumz＠svn.wikimedia.org 04 May '09

04 May '09

Revision: 6816 Author: nicdumz Date: 2009-05-04 01:26:35 +0000 (Mon, 04 May 2009) Log Message: ----------- pywikibot.Error needs a basestring argument on init. Modified Paths: -------------- branches/rewrite/scripts/interwiki.py Modified: branches/rewrite/scripts/interwiki.py =================================================================== --- branches/rewrite/scripts/interwiki.py 2009-05-04 01:20:36 UTC (rev 6815) +++ branches/rewrite/scripts/interwiki.py 2009-05-04 01:26:35 UTC (rev 6816) @@ -1446,17 +1446,17 @@ if globalvar.localonly: # In this case only continue on the Page we started with if page != self.originPage: - raise SaveError + raise SaveError(u'-localonly and page != originPage') if page.section(): # This is not a page, but a subpage. Do not edit it. pywikibot.output(u"Not editing %s: not doing interwiki on subpages" % page) - raise SaveError + raise SaveError(u'Link has a #section') try: pagetext = page.get() except pywikibot.NoPage: pywikibot.output(u"Not editing %s: page does not exist" % page) - raise SaveError + raise SaveError(u'Page doesn\'t exist') # Show a message in purple. pywikibot.output(u"\03{lightpurple}Updating links on page %s.\03{default}" % page) @@ -1490,7 +1490,7 @@ s = u"None" if pltmp is not None: s = pltmp pywikibot.output(u"BUG>>> %s is not in the list of new links! Found %s." % (page, s)) - raise SaveError + raise SaveError(u'BUG: sanity check failed') # Avoid adding an iw link back to itself del new[page.site()] @@ -1580,16 +1580,16 @@ status, reason, data = page.put(newtext, comment = pywikibot.translate(page.site().lang, msg)[0] + mods) except pywikibot.LockedPage: pywikibot.output(u'Page %s is locked. Skipping.' % (page,)) - raise SaveError + raise SaveError(u'Locked') except pywikibot.EditConflict: pywikibot.output(u'ERROR putting page: An edit conflict occurred. Giving up.') - raise SaveError + raise SaveError(u'Edit conflict') except (pywikibot.SpamfilterError), error: pywikibot.output(u'ERROR putting page: %s blacklisted by spamfilter. Giving up.' % (error.url,)) - raise SaveError + raise SaveError(u'Spam filter') except (pywikibot.PageNotSaved), error: pywikibot.output(u'ERROR putting page: %s' % (error.args,)) - raise SaveError + raise SaveError(u'PageNotSaved') except (socket.error, IOError), error: if timeout>3600: raise @@ -1612,7 +1612,7 @@ pywikibot.output(u'%s %s' % (status, reason)) return False elif answer == 'g': - raise GiveUpOnPage + raise GiveUpOnPage(u'User asked us to give up') else: raise LinkMustBeRemoved(u'Found incorrect link to %s in %s'% (",".join([x.lang for x in removing]), page))

1 0

SVN: [6815] branches/rewrite/scripts/interwiki.py
by nicdumz＠svn.wikimedia.org 04 May '09

04 May '09

Revision: 6815 Author: nicdumz Date: 2009-05-04 01:20:36 +0000 (Mon, 04 May 2009) Log Message: ----------- sectionFreeTitle is obsolete, using title instead Modified Paths: -------------- branches/rewrite/scripts/interwiki.py Modified: branches/rewrite/scripts/interwiki.py =================================================================== --- branches/rewrite/scripts/interwiki.py 2009-05-04 01:19:22 UTC (rev 6814) +++ branches/rewrite/scripts/interwiki.py 2009-05-04 01:20:36 UTC (rev 6815) @@ -1448,7 +1448,7 @@ if page != self.originPage: raise SaveError - if page.title() != page.sectionFreeTitle(): + if page.section(): # This is not a page, but a subpage. Do not edit it. pywikibot.output(u"Not editing %s: not doing interwiki on subpages" % page) raise SaveError

1 0

SVN: [6814] branches/rewrite/scripts/interwiki.py
by nicdumz＠svn.wikimedia.org 04 May '09

04 May '09

Revision: 6814 Author: nicdumz Date: 2009-05-04 01:19:22 +0000 (Mon, 04 May 2009) Log Message: ----------- Use Site.preloadpages(templates=True, langlinks=True) Modified Paths: -------------- branches/rewrite/scripts/interwiki.py Modified: branches/rewrite/scripts/interwiki.py =================================================================== --- branches/rewrite/scripts/interwiki.py 2009-05-04 01:18:44 UTC (rev 6813) +++ branches/rewrite/scripts/interwiki.py 2009-05-04 01:19:22 UTC (rev 6814) @@ -1828,7 +1828,7 @@ pywikibot.output(u"NOTE: Nothing left to do 2") return False # Get the content of the assembled list in one blow - gen = pagegenerators.PreloadingGenerator(iter(pageGroup)) + gen = site.preloadpages(pageGroup, templates=True, langlinks=True) for page in gen: # we don't want to do anything with them now. The # page contents will be read via the Subject class.

1 0

SVN: [6813] branches/rewrite/scripts/interwiki.py
by nicdumz＠svn.wikimedia.org 04 May '09

04 May '09

Revision: 6813 Author: nicdumz Date: 2009-05-04 01:18:44 +0000 (Mon, 04 May 2009) Log Message: ----------- Using a pickle file instead of a plaintext file for interwiki dumps Putting data in data/interwiki-dumps instead of ./interwiki-dumps Modified Paths: -------------- branches/rewrite/scripts/interwiki.py Modified: branches/rewrite/scripts/interwiki.py =================================================================== --- branches/rewrite/scripts/interwiki.py 2009-05-04 01:17:45 UTC (rev 6812) +++ branches/rewrite/scripts/interwiki.py 2009-05-04 01:18:44 UTC (rev 6813) @@ -279,7 +279,7 @@ import sys, copy, re, os import time -import codecs +import codecs, pickle import socket try: @@ -1692,11 +1692,12 @@ def dump(self): site = pywikibot.getSite() dumpfn = pywikibot.config.datafilepath( + 'data', 'interwiki-dumps', - 'interwikidump-%s-%s.txt' % (site.family.name, site.lang)) - f = codecs.open(dumpfn, 'w', 'utf-8') - for subj in self.subjects: - f.write(unicode(subj.originPage)+'\n') + '%s-%s.pickle' % (site.family.name, site.lang)) + f = open(dumpfn, 'w') + titles = [s.originPage.title() for s in self.subjects] + pickle.dump(titles, f) f.close() pywikibot.output(u'Dump %s (%s) saved' % (site.lang, site.family.name)) @@ -2105,24 +2106,29 @@ if optRestore or optContinue: site = pywikibot.getSite() dumpFileName = pywikibot.config.datafilepath( + 'data', 'interwiki-dumps', - u'interwikidump-%s-%s.txt' + u'%s-%s.pickle' % (site.family.name, site.lang)) - hintlessPageGen = pagegenerators.TextfilePageGenerator(dumpFileName) + try: + f = open(dumpFileName, 'r') + dumpedTitles = pickle.load(f) + f.close() + except (EOFError, IOError): + dumpedTitles = [] + pages = [pywikibot.Page(site, title) for title in dumpedTitles] + + hintlessPageGen = iter(pages) if optContinue: - # We waste this generator to find out the last page's title - # This is an ugly workaround. - for page in hintlessPageGen: - pass - try: - nextPage = page.titleWithoutNamespace() + '!' - namespace = page.namespace() - except NameError: + if pages: + last = pages[-1] + nextPage = last.title(withNamespace=False) + '!' + namespace = last.namespace() + else: pywikibot.output(u"Dump file is empty?! Starting at the beginning.") nextPage = "!" namespace = 0 - # old generator is used up, create a new one - hintlessPageGen = pagegenerators.CombinedPageGenerator([pagegenerators.TextfilePageGenerator(dumpFileName), pagegenerators.AllpagesPageGenerator(nextPage, namespace, includeredirects = False)]) + hintlessPageGen = pagegenerators.CombinedPageGenerator([hintlessPageGen, pagegenerators.AllpagesPageGenerator(nextPage, namespace, includeredirects = False)]) bot = InterwikiBot()

1 0

SVN: [6812] branches/rewrite/scripts/interwiki.py
by nicdumz＠svn.wikimedia.org 04 May '09

04 May '09

Revision: 6812 Author: nicdumz Date: 2009-05-04 01:17:45 +0000 (Mon, 04 May 2009) Log Message: ----------- interwiki() -> (iter|)langlinks() Conversion is naive, converting Link into Page: it might be more interesting to use Links sometimes Modified Paths: -------------- branches/rewrite/scripts/interwiki.py Modified: branches/rewrite/scripts/interwiki.py =================================================================== --- branches/rewrite/scripts/interwiki.py 2009-05-04 01:16:16 UTC (rev 6811) +++ branches/rewrite/scripts/interwiki.py 2009-05-04 01:17:45 UTC (rev 6812) @@ -1113,7 +1113,7 @@ # Page exists, isnt a redirect, and is a plain link (no section) try: - iw = page.interwiki() + iw = page.langlinks() except pywikibot.NoSuchSite: pywikibot.output(u"NOTE: site %s does not exist" % page.site()) continue @@ -1169,7 +1169,8 @@ self.done.remove(page) iw = () - for linkedPage in iw: + for link in iw: + linkedPage = pywikibot.Page(link) if globalvar.hintsareright: if linkedPage.site in self.hintedsites: pywikibot.output(u"NOTE: %s: %s extra interwiki on hinted site ignored %s" % (self.originPage, page, linkedPage)) @@ -1377,7 +1378,8 @@ elif not globalvar.strictlimittwo and site in new and site != lclSite: old={} try: - for page in new[site].interwiki(): + for link in new[site].iterlanglinks(): + page = pywikibot.Page(link) old[page.site()] = page except pywikibot.NoPage: pywikibot.output(u"BUG>>> %s no longer exists?" % new[site]) @@ -1462,7 +1464,7 @@ # clone original newPages dictionary, so that we can modify it to the local page's needs new = dict(newPages) - interwikis = page.interwiki() + interwikis = [pywikibot.Page(l) for l in page.iterlanglinks()] # remove interwiki links to ignore for iw in re.finditer('', pagetext): @@ -1630,7 +1632,7 @@ page = new[site] if not page.section(): try: - linkedPages = set(page.interwiki()) + linkedPages = set(pywikibot.Page(l) for l in page.iterlanglinks()) except pywikibot.NoPage: pywikibot.output(u"WARNING: Page %s does no longer exist?!" % page) break

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn