Pywikipedia-l December 2007

pywikipedia-l@lists.wikimedia.org

26 participants
320 discussions

SVN: [4748] trunk/pywikipedia/checkimages.py
by filnik＠svn.wikimedia.org 22 Dec '07

22 Dec '07

Revision: 4748 Author: filnik Date: 2007-12-22 13:06:46 +0000 (Sat, 22 Dec 2007) Log Message: ----------- Adding a \n in the commons' message to prevent errors Modified Paths: -------------- trunk/pywikipedia/checkimages.py Modified: trunk/pywikipedia/checkimages.py =================================================================== --- trunk/pywikipedia/checkimages.py 2007-12-22 10:27:48 UTC (rev 4747) +++ trunk/pywikipedia/checkimages.py 2007-12-22 13:06:46 UTC (rev 4748) @@ -166,7 +166,7 @@ } # That's the text that the bot will add if it doesn't find the license. nothing_notification = { - 'commons':"{{subst:User:Filnik/untagged|Image:%s}}\n\n''This message was '''added automatically by [[User:Filbot|Filbot]]''', if you need some help about it, ask [[User:Filnik|its master]] or go to the [[Commons:Help desk]]''. --~~~~", + 'commons':"\n{{subst:User:Filnik/untagged|Image:%s}}\n\n''This message was '''added automatically by [[User:Filbot|Filbot]]''', if you need some help about it, ask [[User:Filnik|its master]] or go to the [[Commons:Help desk]]''. --~~~~", 'en' :"{{subst:image source|Image:%s}} --~~~~", 'it' :"{{subst:Utente:Filbot/Senza licenza|%s}} --~~~~", 'hu' :u"{{subst:adjforrást|Kép:%s}} \n Ezt az üzenetet ~~~ automatikusan helyezte el a vitalapodon, kérdéseddel fordulj a gazdájához, vagy a [[WP:KF|Kocsmafalhoz]]. --~~~~",

1 0

SVN: [4747] trunk/pywikipedia/category.py
by rotem＠svn.wikimedia.org 22 Dec '07

22 Dec '07

Revision: 4747 Author: rotem Date: 2007-12-22 10:27:48 +0000 (Sat, 22 Dec 2007) Log Message: ----------- Fixing a variable name. Modified Paths: -------------- trunk/pywikipedia/category.py Modified: trunk/pywikipedia/category.py =================================================================== --- trunk/pywikipedia/category.py 2007-12-22 10:26:24 UTC (rev 4746) +++ trunk/pywikipedia/category.py 2007-12-22 10:27:48 UTC (rev 4747) @@ -401,7 +401,7 @@ preloadingGen = pagegenerators.PreloadingGenerator(gen) for subcategory in preloadingGen: if not self.titleRegex or re.search(self.titleRegex,subcategory.title()): - catlib.change_category(article, self.oldCat, newCat, inPlace=self.inPlace) + catlib.change_category(subcategory, self.oldCat, newCat, inPlace=self.inPlace) # Delete the old category and its moved talk page if copied and self.deleteEmptySourceCat == True:

1 0

SVN: [4746] trunk/pywikipedia/category.py
by rotem＠svn.wikimedia.org 22 Dec '07

22 Dec '07

Revision: 4746 Author: rotem Date: 2007-12-22 10:26:24 +0000 (Sat, 22 Dec 2007) Log Message: ----------- Removing a debug line. Modified Paths: -------------- trunk/pywikipedia/category.py Modified: trunk/pywikipedia/category.py =================================================================== --- trunk/pywikipedia/category.py 2007-12-22 10:14:39 UTC (rev 4745) +++ trunk/pywikipedia/category.py 2007-12-22 10:26:24 UTC (rev 4746) @@ -380,7 +380,6 @@ oldMovedTalk = None if self.oldCat.exists() and self.moveCatPage: copied = self.oldCat.copyAndKeep(self.newCatTitle, wikipedia.translate(wikipedia.getSite(), cfd_templates)) - copied = True # ALREADY COPIED! # Also move the talk page if copied: reason = wikipedia.translate(wikipedia.getSite(), deletion_reason_move) % (self.newCatTitle, self.newCatTitle)

1 0

SVN: [4745] trunk/pywikipedia
by rotem＠svn.wikimedia.org 22 Dec '07

22 Dec '07

Revision: 4745 Author: rotem Date: 2007-12-22 10:14:39 +0000 (Sat, 22 Dec 2007) Log Message: ----------- Avoiding duplicate code in the query of Special:Linksearch. Modified Paths: -------------- trunk/pywikipedia/family.py trunk/pywikipedia/pagegenerators.py trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/family.py =================================================================== --- trunk/pywikipedia/family.py 2007-12-21 21:28:46 UTC (rev 4744) +++ trunk/pywikipedia/family.py 2007-12-22 10:14:39 UTC (rev 4745) @@ -2721,7 +2721,7 @@ return '%s?title=%s:Ipblocklist&action=search&ip=%s' % (self.path(code), self.special_namespace_url(code), name) def linksearch_address(self, code, link, limit=500, offset=0): - return '%s?title=%s:Linksearch&limit=%d&offset=%d&target=%s' % (self.path(code), self.special_namespace_url(code), limit, offset, link) + return '%s?title=%s:Linksearch&limit=%d&offset=%d&target=%s' % (self.path(code), self.special_namespace_url(code), limit, offset, link) def version_history_address(self, code, name, limit = config.special_page_limit): return '%s?title=%s&action=history&limit=%d' % (self.path(code), name, limit) Modified: trunk/pywikipedia/pagegenerators.py =================================================================== --- trunk/pywikipedia/pagegenerators.py 2007-12-21 21:28:46 UTC (rev 4744) +++ trunk/pywikipedia/pagegenerators.py 2007-12-22 10:14:39 UTC (rev 4745) @@ -299,29 +299,11 @@ def LinksearchPageGenerator(link, step=500, site=None): """Yields all pages that include a specified link, according to [[Special:Linksearch]]. - Retrieves in chunks of size "step" (default 500). - Does not guarantee that resulting pages are unique. """ if site is None: site = wikipedia.getSite() - elRX = re.compile('<a .* class="external ?" .*</a>.*<a .*>(.*)</a>') #TODO: de-uglify? - offset = 0 - pageyeldlist = list() - found = step - while found == step: - found = 0 - url = site.linksearch_address(link,limit=step,offset=offset) - wikipedia.output(u'Querying [[Special:Linksearch]]...') - data = site.getUrl(url) - for elM in elRX.finditer(data): - found += 1 - pagenameofthelink = elM.group(1) - if pagenameofthelink in pageyeldlist: - continue - else: - pageyeldlist.append(pagenameofthelink) - yield wikipedia.Page(site, pagenameofthelink) - offset += step + for page in site.linksearch(link): + yield page def SearchPageGenerator(query, number = 100, namespaces = None, site = None): """ Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-12-21 21:28:46 UTC (rev 4744) +++ trunk/pywikipedia/wikipedia.py 2007-12-22 10:14:39 UTC (rev 4745) @@ -4484,8 +4484,10 @@ """Yield Pages from results of Special:Linksearch for 'siteurl'.""" if siteurl.startswith('*.'): siteurl = siteurl[2:] - for url in [siteurl, "*."+siteurl]: - path = self.family.linksearch_address(self.lang, url) + output(u'Querying [[Special:Linksearch]]...') + cache = [] + for url in [siteurl, '*.' + siteurl]: + path = self.linksearch_address(url) get_throttle() html = self.getUrl(path) loc = html.find('<div class="mw-spcontent">') @@ -4498,7 +4500,11 @@ for title in R.findall(html): if not siteurl in title: # the links themselves have similar form - yield Page(self,title) + if title in cache: + continue + else: + cache.append(title) + yield Page(self, title) def __repr__(self): return self.family.name+":"+self.lang

1 0

SVN: [4744] trunk/pywikipedia/families/wikipedia_family.py
by malafaya＠svn.wikimedia.org 22 Dec '07

22 Dec '07

Revision: 4744 Author: malafaya Date: 2007-12-21 21:28:46 +0000 (Fri, 21 Dec 2007) Log Message: ----------- some namespaces for ro, hr, and oc wikipedias Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2007-12-21 18:58:54 UTC (rev 4743) +++ trunk/pywikipedia/families/wikipedia_family.py 2007-12-21 21:28:46 UTC (rev 4744) @@ -279,6 +279,7 @@ 'new': u'दबू', 'nl': u'Portaal', 'no': u'Portal', + 'oc': u'Portal', 'pl': u'Portal', 'pt': u'Portal', 'ro': u'Portal', @@ -336,6 +337,7 @@ 'new': u'दबू खँलाबँला', 'nl': u'Overleg portaal', 'no': u'Portaldiskusjon', + 'oc': u'Discussion Portal', 'pl': u'Dyskusja portalu', 'pt': [u'Portal Discussão', u'Discussão Portal'], 'ro': u'Discuţie Portal', @@ -364,10 +366,13 @@ 'eu': u'Wikiproiektu', 'fi': u'Metasivu', 'fr': u'Projet', + 'hr': u'Dodatak', 'it': u'Progetto', 'lmo': u'Purtaal', + 'oc': u'Projècte', 'pl': u'Wikiprojekt', 'pt': u'Anexo', + 'ro': u'Proiect', } self.namespaces[103] = { @@ -377,10 +382,13 @@ 'eu': u'Wikiproiektu eztabaida', 'fi': u'Keskustelu metasivusta', 'fr': u'Discussion Projet', + 'hr': u'Razgovor o dodatku', 'it': u'Discussioni progetto', 'lmo': u'Descüssiun Purtaal', + 'oc': u'Discussion Projècte', 'pl': u'Dyskusja Wikiprojektu', 'pt': u'Anexo Discussão', + 'ro': u'Discuţie Proiect', } self.namespaces[104] = {

1 0

SVN: [4743] trunk/pywikipedia
by rotem＠svn.wikimedia.org 22 Dec '07

22 Dec '07

Revision: 4743 Author: rotem Date: 2007-12-21 18:58:54 +0000 (Fri, 21 Dec 2007) Log Message: ----------- Creating wikipedia.site.prefixindex (which still uses wikipedia.site.allpages), and making pagegenerators.PrefixingPageGenerator use it. Modified Paths: -------------- trunk/pywikipedia/pagegenerators.py trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/pagegenerators.py =================================================================== --- trunk/pywikipedia/pagegenerators.py 2007-12-21 18:46:24 UTC (rev 4742) +++ trunk/pywikipedia/pagegenerators.py 2007-12-21 18:58:54 UTC (rev 4743) @@ -117,18 +117,22 @@ """ if site is None: site = wikipedia.getSite() + page = wikipedia.Page(site, start) if namespace is None: - namespace = wikipedia.Page(site, start).namespace() - title = wikipedia.Page(site, start).titleWithoutNamespace() - for page in site.allpages(start=title, namespace=namespace, includeredirects = includeredirects): + namespace = page.namespace() + title = page.titleWithoutNamespace() + for page in site.allpages(start = title, namespace = namespace, includeredirects = includeredirects): yield page def PrefixingPageGenerator(prefix, namespace = None, includeredirects = True, site = None): - for page in AllpagesPageGenerator(prefix, namespace, includeredirects, site): - if page.titleWithoutNamespace().startswith(prefix): - yield page - else: - break + if site is None: + site = wikipedia.getSite() + page = wikipedia.Page(site, prefix) + if namespace is None: + namespace = page.namespace() + title = page.titleWithoutNamespace() + for page in site.prefixindex(prefix = title, namespace = namespace, includeredirects = includeredirects): + yield page def NewpagesPageGenerator(number = 100, get_redirect = False, repeat = False, site = None): if site is None: Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-12-21 18:46:24 UTC (rev 4742) +++ trunk/pywikipedia/wikipedia.py 2007-12-21 18:58:54 UTC (rev 4743) @@ -3491,6 +3491,7 @@ search(query): query results from Special:Search allpages(): Special:Allpages + prefixindex(): Special:Prefixindex newpages(): Special:Newpages newimages(): Special:Log&type=upload longpages(): Special:Longpages @@ -4456,6 +4457,29 @@ else: break + def prefixindex(self, prefix, namespace=0, includeredirects=True): + """Yield all pages with a given prefix. + + Parameters: + prefix The prefix of the pages. + namespace Namespace number; defaults to 0. + MediaWiki software will only return pages in one namespace + at a time. + + If includeredirects is False, redirects will not be found. + If includeredirects equals the string 'only', only redirects + will be found. Note that this has not been tested on older + versions of the MediaWiki code. + + It is advised not to use this directly, but to use the + PrefixingPageGenerator from pagegenerators.py instead. + """ + for page in self.allpages(start = prefix, namespace = namespace, includeredirects = includeredirects): + if page.titleWithoutNamespace().startswith(prefix): + yield page + else: + break + def linksearch(self, siteurl): """Yield Pages from results of Special:Linksearch for 'siteurl'.""" if siteurl.startswith('*.'):

1 0

SVN: [4742] trunk/pywikipedia/fixes.py
by wikipedian＠svn.wikimedia.org 22 Dec '07

22 Dec '07

Revision: 4742 Author: wikipedian Date: 2007-12-21 18:46:24 +0000 (Fri, 21 Dec 2007) Log Message: ----------- ISBN fixes: unicode bugfix Modified Paths: -------------- trunk/pywikipedia/fixes.py Modified: trunk/pywikipedia/fixes.py =================================================================== --- trunk/pywikipedia/fixes.py 2007-12-20 18:38:39 UTC (rev 4741) +++ trunk/pywikipedia/fixes.py 2007-12-21 18:46:24 UTC (rev 4742) @@ -336,7 +336,7 @@ # Note that these regular expressions also match valid ISBNs, but # these won't be changed. (ur'ISBN (978|979) *[\- –\.] *(\d+) *[\- –\.] *(\d+) *[\- –\.] *(\d+) *[\- –\.] *(\d)(?!\d)', r'ISBN \1-\2-\3-\4-\5'), # ISBN13 - (r'ISBN (\d+) *[\- –\.] *(\d+) *[\- –\.] *(\d+) *[\- –\.] *(\d|X|x)(?!\d)', r'ISBN \1-\2-\3-\4'), # ISBN10 + (ur'ISBN (\d+) *[\- –\.] *(\d+) *[\- –\.] *(\d+) *[\- –\.] *(\d|X|x)(?!\d)', r'ISBN \1-\2-\3-\4'), # ISBN10 ], 'exceptions': { 'inside-tags': [

1 0

[ pywikipediabot-Bugs-1855071 ] "redirect.py double -xml:xx -namespace:x" crashing
by SourceForge.net 21 Dec '07

21 Dec '07

Bugs item #1855071, was opened at 2007-12-20 13:10 Message generated for change (Comment added) made by russblau You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1855071&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None Status: Open Resolution: None Priority: 5 Private: No Submitted By: Nicolas Dumazet (nicdumz) Assigned to: Nobody/Anonymous (nobody) Summary: "redirect.py double -xml:xx -namespace:x" crashing Initial Comment: It is not even loading the XML, i get an instant error : python redirect.py double -namespace:0 -xml:frwiki-20071203-pages-articles.xml Checked for running processes. 1 processes currently running, including the current process. Reading XML dump... Traceback (most recent call last): File "redirect.py", line 377, in <module> main() File "redirect.py", line 373, in main bot.run() File "redirect.py", line 328, in run self.fix_double_redirects() File "redirect.py", line 255, in fix_double_redirects for redir_name in self.generator.retrieve_double_redirects(): File "redirect.py", line 199, in retrieve_double_redirects dict = self.get_redirects_from_dump() File "redirect.py", line 123, in get_redirects_from_dump if self.namespace and self.namespace != entry.namespace: AttributeError: XmlEntry instance has no attribute 'namespace' ---------------------------------------------------------------------- Comment By: Russell Blau (russblau) Date: 2007-12-20 13:40 Message: Logged In: YES user_id=855050 Originator: NO fixed in r4741 ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1855071&group_…

1 0

[ pywikipediabot-Bugs-1855044 ] redirect.py crashes when finding a bad page
by SourceForge.net 21 Dec '07

21 Dec '07

Bugs item #1855044, was opened at 2007-12-20 12:20 Message generated for change (Comment added) made by russblau You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1855044&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None Status: Open Resolution: None Priority: 5 Private: No Submitted By: Nicolas Dumazet (nicdumz) Assigned to: Nobody/Anonymous (nobody) Summary: redirect.py crashes when finding a bad page Initial Comment: When doing a simple "redirect.py double -xml:xx.xml", I got : Traceback (most recent call last): File "redirect.py", line 377, in <module> main() File "redirect.py", line 373, in main bot.run() File "redirect.py", line 328, in run self.fix_double_redirects() File "redirect.py", line 273, in fix_double_redirects secondTargetPage = secondRedir.getRedirectTarget() File "/home/nico/projets/pywikipedia/wikipedia.py", line 1576, in getRedirectTarget self.get() File "/home/nico/projets/pywikipedia/wikipedia.py", line 595, in get self._contents, self._isWatched, self.editRestriction = self._getEditPage(get_redirect = get_redirect, throttle = throttle, sysop = sysop, nofollow_redirects=nofollow_redirects) File "/home/nico/projets/pywikipedia/wikipedia.py", line 679, in _getEditPage raise BadTitle('BadTitle: %s' % self) wikipedia.BadTitle: BadTitle: [[../Projet/Sciences/Champs magnétiques B et H]] Now, the redirect page was : #REDIRECT [[../Projet/Sciences/Champs magnétiques B et H]], which is not correct. But I think that redirect.py is supposed to ask the user if not skipping it, instead of crashing ;) Thanks, Nicolas Dumazet. ---------------------------------------------------------------------- Comment By: Russell Blau (russblau) Date: 2007-12-20 13:39 Message: Logged In: YES user_id=855050 Originator: NO fixed in r4741 ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1855044&group_…

1 0

SVN: [4741] trunk/pywikipedia/redirect.py
by russblau＠svn.wikimedia.org 21 Dec '07

21 Dec '07

Revision: 4741 Author: russblau Date: 2007-12-20 18:38:39 +0000 (Thu, 20 Dec 2007) Log Message: ----------- Fix bugs 1855044 and 1855071 Modified Paths: -------------- trunk/pywikipedia/redirect.py Modified: trunk/pywikipedia/redirect.py =================================================================== --- trunk/pywikipedia/redirect.py 2007-12-20 17:09:58 UTC (rev 4740) +++ trunk/pywikipedia/redirect.py 2007-12-20 18:38:39 UTC (rev 4741) @@ -95,9 +95,9 @@ } class RedirectGenerator: - def __init__(self, xmlFilename = None, namespace = None, offset = -1): + def __init__(self, xmlFilename = None, namespaces = None, offset = -1): self.xmlFilename = xmlFilename - self.namespace = namespace + self.namespaces = namespaces self.offset = offset def get_redirects_from_dump(self, alsoGetPageTitles = False): @@ -111,7 +111,8 @@ dict = {} # open xml dump and read page titles out of it dump = xmlreader.XmlDump(xmlFilename) - redirR = wikipedia.getSite().redirectRegex() + site = wikipedia.getSite() + redirR = site.redirectRegex() readPagesCount = 0 if alsoGetPageTitles: pageTitles = set() @@ -120,8 +121,9 @@ # always print status message after 10000 pages if readPagesCount % 10000 == 0: wikipedia.output(u'%i pages read...' % readPagesCount) - if self.namespace and self.namespace != entry.namespace: - continue + if self.namespaces is not None: + if wikipedia.Page(site, entry.title).namespace() not in self.namespaces: + continue if alsoGetPageTitles: pageTitles.add(entry.title.replace(' ', '_')) @@ -129,7 +131,7 @@ if m: target = m.group(1) # There might be redirects to another wiki. Ignore these. - for code in wikipedia.getSite().family.langs.keys(): + for code in site.family.langs.keys(): if target.startswith('%s:' % code) or target.startswith(':%s:' % code): wikipedia.output(u'NOTE: Ignoring %s which is a redirect to %s:' % (entry.title, code)) target = None @@ -279,6 +281,11 @@ wikipedia.output( u'Redirect target %s is not a redirect.' % secondRedir.aslink()) + except wikipedia.BadTitle, e: + # str(e) is in the format 'BadTitle: [[Foo]]' + wikipedia.output( + u'Redirect target %s is not a valid page title.' + % str(e)[10:]) except wikipedia.NoPage: wikipedia.output( u'Redirect target %s doesn\'t exist.'

1 0

← Newer
1
...
5
6
7
8
9
10
11
...
32
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

Pywikipedia-l December 2007