Revision: 6134 Author: russblau Date: 2008-12-08 21:41:15 +0000 (Mon, 08 Dec 2008) Log Message: ----------- Interwiki link scraping Modified Paths: -------------- branches/rewrite/pywikibot/data/api.py branches/rewrite/pywikibot/page.py branches/rewrite/pywikibot/site.py branches/rewrite/pywikibot/tests/page_tests.py Modified: branches/rewrite/pywikibot/data/api.py =================================================================== --- branches/rewrite/pywikibot/data/api.py 2008-12-05 22:08:29 UTC (rev 6133) +++ branches/rewrite/pywikibot/data/api.py 2008-12-08 21:41:15 UTC (rev 6134) @@ -184,7 +184,7 @@ self.site.throttle(write=write) uri = self.site.scriptpath() + "/api.php" try: - if write or action == "login": + if write or action in ("login", "expandtemplates", "parse"): # add other actions that require POST requests above rawdata = http.request(self.site, uri, method="POST", headers={'Content-Type': Modified: branches/rewrite/pywikibot/page.py =================================================================== --- branches/rewrite/pywikibot/page.py 2008-12-05 22:08:29 UTC (rev 6133) +++ branches/rewrite/pywikibot/page.py 2008-12-08 21:41:15 UTC (rev 6134) @@ -368,6 +368,15 @@ text = property(_textgetter, _textsetter, _cleartext, "The edited wikitext (unicode) of this Page") + def expand_text(self): + """Return the page text with all templates expanded.""" + req = pywikibot.data.api.Request(action="expandtemplates", + text=self.text, + title=self.title(withSection=False), + site=self.site()) + result = req.submit() + return result["expandtemplates"]["*"] + def userName(self): """Return name or IP address of last user to edit page.""" return self._revisions[self.latestRevision()].user @@ -686,8 +695,8 @@ """Iterate Pages that this Page links to. Only returns pages from "normal" internal links. Image and category - links are omitted unless prefixed with ":"; embedded templates are - omitted (but links within them are returned); all interwiki and + links are omitted unless prefixed with ":". Embedded templates are + omitted (but links within them are returned). All interwiki and external links are omitted. @return: a generator that yields Page objects. @@ -695,20 +704,43 @@ """ return self.site().pagelinks(self) - def interwiki(self): - """Iterate interwiki links in the page text. + def interwiki(self, expand=True): + """Iterate interwiki links in the page text, excluding language links. - @return: a generator that yields Link objects. + @param expand: if True (default), include interwiki links found in + templates transcluded onto this page; if False, only iterate + interwiki links found in this page's own wikitext + @return: a generator that yields Link objects """ - return self.site().pageinterwiki(self) + # This function does not exist in the API, so it has to be + # implemented by screen-scraping + Rlink = re.compile(r'\[\[(?P<title>[^\]|[#<>{}]*)(\|.*?)?\]\]') + if expand: + text = self.expand_text() + else: + text = self.text + for linkmatch in Rlink.finditer( + pywikibot.textlib.removeDisabledParts(text)): + linktitle = linkmatch.group("title") + link = Link(linktitle, self.site()) + # only yield links that are to a different site and that + # are not language links + try: + if link.site != self.site(): + if linktitle.lstrip().startswith(":"): + # initial ":" indicates not a language link + yield link + elif link.site.family != self.site().family: + # link to a different family is not a language link + yield link + except pywikibot.Error: + # ignore any links with invalid contents + continue def langlinks(self): """Iterate all interlanguage links on this page. - Note that the links yielded by this method will be a subset of - the results of self.interwiki(). - @return: a generator that yields Link objects. """ @@ -1729,7 +1761,25 @@ def __str__(self): return self.astext() + def __cmp__(self, other): + """Test for equality and inequality of Link objects. + Link objects are "equal" if and only if they are on the same site + and have the same normalized title, including section if any. + + Link objects are sortable by site, then namespace, then title. + + """ + if not isinstance(other, Link): + # especially, return -1 if other is None + return -1 + if not self.site == other.site: + return cmp(self.site, other.site) + if self.namespace != other.namespace: + return cmp(self.namespace, other.namespace) + return cmp(self.title, other.title) + + # Utility functions for parsing page titles def html2unicode(text, ignore = []): @@ -1794,7 +1844,7 @@ unicodeCodepoint=convertIllegalHtmlEntities[unicodeCodepoint] except KeyError: pass - if unicodeCodepoint and unicodeCodepoint not in ignore and (WIDEBUILD or unicodeCodepoint < 65534): + if unicodeCodepoint and unicodeCodepoint not in ignore: result += unichr(unicodeCodepoint) else: # Leave the entity unchanged Modified: branches/rewrite/pywikibot/site.py =================================================================== --- branches/rewrite/pywikibot/site.py 2008-12-05 22:08:29 UTC (rev 6133) +++ branches/rewrite/pywikibot/site.py 2008-12-08 21:41:15 UTC (rev 6134) @@ -1357,7 +1357,7 @@ api.update_page(page, pagedata) def pageinterwiki(self, page): - # TODO + # No such function in the API (this method isn't called anywhere) raise NotImplementedError def pagelanglinks(self, page): Modified: branches/rewrite/pywikibot/tests/page_tests.py =================================================================== --- branches/rewrite/pywikibot/tests/page_tests.py 2008-12-05 22:08:29 UTC (rev 6133) +++ branches/rewrite/pywikibot/tests/page_tests.py 2008-12-08 21:41:15 UTC (rev 6134) @@ -227,9 +227,12 @@ def testLinks(self): for p in mainpage.linkedPages(): self.assertTrue(isinstance(p, pywikibot.Page)) -## Not implemented: -## for p in mainpage.interwiki(): -## self.assertTrue(isinstance(p, pywikibot.Link)) + iw = list(mainpage.interwiki(expand=True)) + for p in iw: + self.assertTrue(isinstance(p, pywikibot.Link)) + for p2 in mainpage.interwiki(expand=False): + self.assertTrue(isinstance(p2, pywikibot.Link)) + self.assertTrue(p2 in iw) for p in mainpage.langlinks(): self.assertTrue(isinstance(p, pywikibot.Link)) for p in mainpage.imagelinks():