pywikibot

pywikibot@lists.wikimedia.org

3 participants
6830 discussions

[Pywikipedia-l] SVN: [4744] trunk/pywikipedia/families/wikipedia_family.py
by malafaya＠svn.wikimedia.org 21 Dec '07

21 Dec '07

Revision: 4744 Author: malafaya Date: 2007-12-21 21:28:46 +0000 (Fri, 21 Dec 2007) Log Message: ----------- some namespaces for ro, hr, and oc wikipedias Modified Paths: -------------- trunk/pywikipedia/families/wikipedia_family.py Modified: trunk/pywikipedia/families/wikipedia_family.py =================================================================== --- trunk/pywikipedia/families/wikipedia_family.py 2007-12-21 18:58:54 UTC (rev 4743) +++ trunk/pywikipedia/families/wikipedia_family.py 2007-12-21 21:28:46 UTC (rev 4744) @@ -279,6 +279,7 @@ 'new': u'दबू', 'nl': u'Portaal', 'no': u'Portal', + 'oc': u'Portal', 'pl': u'Portal', 'pt': u'Portal', 'ro': u'Portal', @@ -336,6 +337,7 @@ 'new': u'दबू खँलाबँला', 'nl': u'Overleg portaal', 'no': u'Portaldiskusjon', + 'oc': u'Discussion Portal', 'pl': u'Dyskusja portalu', 'pt': [u'Portal Discussão', u'Discussão Portal'], 'ro': u'Discuţie Portal', @@ -364,10 +366,13 @@ 'eu': u'Wikiproiektu', 'fi': u'Metasivu', 'fr': u'Projet', + 'hr': u'Dodatak', 'it': u'Progetto', 'lmo': u'Purtaal', + 'oc': u'Projècte', 'pl': u'Wikiprojekt', 'pt': u'Anexo', + 'ro': u'Proiect', } self.namespaces[103] = { @@ -377,10 +382,13 @@ 'eu': u'Wikiproiektu eztabaida', 'fi': u'Keskustelu metasivusta', 'fr': u'Discussion Projet', + 'hr': u'Razgovor o dodatku', 'it': u'Discussioni progetto', 'lmo': u'Descüssiun Purtaal', + 'oc': u'Discussion Projècte', 'pl': u'Dyskusja Wikiprojektu', 'pt': u'Anexo Discussão', + 'ro': u'Discuţie Proiect', } self.namespaces[104] = {

1 0

[Pywikipedia-l] SVN: [4743] trunk/pywikipedia
by rotem＠svn.wikimedia.org 21 Dec '07

21 Dec '07

Revision: 4743 Author: rotem Date: 2007-12-21 18:58:54 +0000 (Fri, 21 Dec 2007) Log Message: ----------- Creating wikipedia.site.prefixindex (which still uses wikipedia.site.allpages), and making pagegenerators.PrefixingPageGenerator use it. Modified Paths: -------------- trunk/pywikipedia/pagegenerators.py trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/pagegenerators.py =================================================================== --- trunk/pywikipedia/pagegenerators.py 2007-12-21 18:46:24 UTC (rev 4742) +++ trunk/pywikipedia/pagegenerators.py 2007-12-21 18:58:54 UTC (rev 4743) @@ -117,18 +117,22 @@ """ if site is None: site = wikipedia.getSite() + page = wikipedia.Page(site, start) if namespace is None: - namespace = wikipedia.Page(site, start).namespace() - title = wikipedia.Page(site, start).titleWithoutNamespace() - for page in site.allpages(start=title, namespace=namespace, includeredirects = includeredirects): + namespace = page.namespace() + title = page.titleWithoutNamespace() + for page in site.allpages(start = title, namespace = namespace, includeredirects = includeredirects): yield page def PrefixingPageGenerator(prefix, namespace = None, includeredirects = True, site = None): - for page in AllpagesPageGenerator(prefix, namespace, includeredirects, site): - if page.titleWithoutNamespace().startswith(prefix): - yield page - else: - break + if site is None: + site = wikipedia.getSite() + page = wikipedia.Page(site, prefix) + if namespace is None: + namespace = page.namespace() + title = page.titleWithoutNamespace() + for page in site.prefixindex(prefix = title, namespace = namespace, includeredirects = includeredirects): + yield page def NewpagesPageGenerator(number = 100, get_redirect = False, repeat = False, site = None): if site is None: Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-12-21 18:46:24 UTC (rev 4742) +++ trunk/pywikipedia/wikipedia.py 2007-12-21 18:58:54 UTC (rev 4743) @@ -3491,6 +3491,7 @@ search(query): query results from Special:Search allpages(): Special:Allpages + prefixindex(): Special:Prefixindex newpages(): Special:Newpages newimages(): Special:Log&type=upload longpages(): Special:Longpages @@ -4456,6 +4457,29 @@ else: break + def prefixindex(self, prefix, namespace=0, includeredirects=True): + """Yield all pages with a given prefix. + + Parameters: + prefix The prefix of the pages. + namespace Namespace number; defaults to 0. + MediaWiki software will only return pages in one namespace + at a time. + + If includeredirects is False, redirects will not be found. + If includeredirects equals the string 'only', only redirects + will be found. Note that this has not been tested on older + versions of the MediaWiki code. + + It is advised not to use this directly, but to use the + PrefixingPageGenerator from pagegenerators.py instead. + """ + for page in self.allpages(start = prefix, namespace = namespace, includeredirects = includeredirects): + if page.titleWithoutNamespace().startswith(prefix): + yield page + else: + break + def linksearch(self, siteurl): """Yield Pages from results of Special:Linksearch for 'siteurl'.""" if siteurl.startswith('*.'):

1 0

[Pywikipedia-l] SVN: [4742] trunk/pywikipedia/fixes.py
by wikipedian＠svn.wikimedia.org 21 Dec '07

21 Dec '07

Revision: 4742 Author: wikipedian Date: 2007-12-21 18:46:24 +0000 (Fri, 21 Dec 2007) Log Message: ----------- ISBN fixes: unicode bugfix Modified Paths: -------------- trunk/pywikipedia/fixes.py Modified: trunk/pywikipedia/fixes.py =================================================================== --- trunk/pywikipedia/fixes.py 2007-12-20 18:38:39 UTC (rev 4741) +++ trunk/pywikipedia/fixes.py 2007-12-21 18:46:24 UTC (rev 4742) @@ -336,7 +336,7 @@ # Note that these regular expressions also match valid ISBNs, but # these won't be changed. (ur'ISBN (978|979) *[\- –\.] *(\d+) *[\- –\.] *(\d+) *[\- –\.] *(\d+) *[\- –\.] *(\d)(?!\d)', r'ISBN \1-\2-\3-\4-\5'), # ISBN13 - (r'ISBN (\d+) *[\- –\.] *(\d+) *[\- –\.] *(\d+) *[\- –\.] *(\d|X|x)(?!\d)', r'ISBN \1-\2-\3-\4'), # ISBN10 + (ur'ISBN (\d+) *[\- –\.] *(\d+) *[\- –\.] *(\d+) *[\- –\.] *(\d|X|x)(?!\d)', r'ISBN \1-\2-\3-\4'), # ISBN10 ], 'exceptions': { 'inside-tags': [

1 0

[Pywikipedia-l] [ pywikipediabot-Bugs-1855071 ] "redirect.py double -xml:xx -namespace:x" crashing
by SourceForge.net 20 Dec '07

20 Dec '07

Bugs item #1855071, was opened at 2007-12-20 13:10 Message generated for change (Comment added) made by russblau You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1855071&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None Status: Open Resolution: None Priority: 5 Private: No Submitted By: Nicolas Dumazet (nicdumz) Assigned to: Nobody/Anonymous (nobody) Summary: "redirect.py double -xml:xx -namespace:x" crashing Initial Comment: It is not even loading the XML, i get an instant error : python redirect.py double -namespace:0 -xml:frwiki-20071203-pages-articles.xml Checked for running processes. 1 processes currently running, including the current process. Reading XML dump... Traceback (most recent call last): File "redirect.py", line 377, in <module> main() File "redirect.py", line 373, in main bot.run() File "redirect.py", line 328, in run self.fix_double_redirects() File "redirect.py", line 255, in fix_double_redirects for redir_name in self.generator.retrieve_double_redirects(): File "redirect.py", line 199, in retrieve_double_redirects dict = self.get_redirects_from_dump() File "redirect.py", line 123, in get_redirects_from_dump if self.namespace and self.namespace != entry.namespace: AttributeError: XmlEntry instance has no attribute 'namespace' ---------------------------------------------------------------------- Comment By: Russell Blau (russblau) Date: 2007-12-20 13:40 Message: Logged In: YES user_id=855050 Originator: NO fixed in r4741 ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1855071&group_…

1 0

[Pywikipedia-l] [ pywikipediabot-Bugs-1855044 ] redirect.py crashes when finding a bad page
by SourceForge.net 20 Dec '07

20 Dec '07

Bugs item #1855044, was opened at 2007-12-20 12:20 Message generated for change (Comment added) made by russblau You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1855044&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None Status: Open Resolution: None Priority: 5 Private: No Submitted By: Nicolas Dumazet (nicdumz) Assigned to: Nobody/Anonymous (nobody) Summary: redirect.py crashes when finding a bad page Initial Comment: When doing a simple "redirect.py double -xml:xx.xml", I got : Traceback (most recent call last): File "redirect.py", line 377, in <module> main() File "redirect.py", line 373, in main bot.run() File "redirect.py", line 328, in run self.fix_double_redirects() File "redirect.py", line 273, in fix_double_redirects secondTargetPage = secondRedir.getRedirectTarget() File "/home/nico/projets/pywikipedia/wikipedia.py", line 1576, in getRedirectTarget self.get() File "/home/nico/projets/pywikipedia/wikipedia.py", line 595, in get self._contents, self._isWatched, self.editRestriction = self._getEditPage(get_redirect = get_redirect, throttle = throttle, sysop = sysop, nofollow_redirects=nofollow_redirects) File "/home/nico/projets/pywikipedia/wikipedia.py", line 679, in _getEditPage raise BadTitle('BadTitle: %s' % self) wikipedia.BadTitle: BadTitle: [[../Projet/Sciences/Champs magnétiques B et H]] Now, the redirect page was : #REDIRECT [[../Projet/Sciences/Champs magnétiques B et H]], which is not correct. But I think that redirect.py is supposed to ask the user if not skipping it, instead of crashing ;) Thanks, Nicolas Dumazet. ---------------------------------------------------------------------- Comment By: Russell Blau (russblau) Date: 2007-12-20 13:39 Message: Logged In: YES user_id=855050 Originator: NO fixed in r4741 ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1855044&group_…

1 0

[Pywikipedia-l] SVN: [4741] trunk/pywikipedia/redirect.py
by russblau＠svn.wikimedia.org 20 Dec '07

20 Dec '07

Revision: 4741 Author: russblau Date: 2007-12-20 18:38:39 +0000 (Thu, 20 Dec 2007) Log Message: ----------- Fix bugs 1855044 and 1855071 Modified Paths: -------------- trunk/pywikipedia/redirect.py Modified: trunk/pywikipedia/redirect.py =================================================================== --- trunk/pywikipedia/redirect.py 2007-12-20 17:09:58 UTC (rev 4740) +++ trunk/pywikipedia/redirect.py 2007-12-20 18:38:39 UTC (rev 4741) @@ -95,9 +95,9 @@ } class RedirectGenerator: - def __init__(self, xmlFilename = None, namespace = None, offset = -1): + def __init__(self, xmlFilename = None, namespaces = None, offset = -1): self.xmlFilename = xmlFilename - self.namespace = namespace + self.namespaces = namespaces self.offset = offset def get_redirects_from_dump(self, alsoGetPageTitles = False): @@ -111,7 +111,8 @@ dict = {} # open xml dump and read page titles out of it dump = xmlreader.XmlDump(xmlFilename) - redirR = wikipedia.getSite().redirectRegex() + site = wikipedia.getSite() + redirR = site.redirectRegex() readPagesCount = 0 if alsoGetPageTitles: pageTitles = set() @@ -120,8 +121,9 @@ # always print status message after 10000 pages if readPagesCount % 10000 == 0: wikipedia.output(u'%i pages read...' % readPagesCount) - if self.namespace and self.namespace != entry.namespace: - continue + if self.namespaces is not None: + if wikipedia.Page(site, entry.title).namespace() not in self.namespaces: + continue if alsoGetPageTitles: pageTitles.add(entry.title.replace(' ', '_')) @@ -129,7 +131,7 @@ if m: target = m.group(1) # There might be redirects to another wiki. Ignore these. - for code in wikipedia.getSite().family.langs.keys(): + for code in site.family.langs.keys(): if target.startswith('%s:' % code) or target.startswith(':%s:' % code): wikipedia.output(u'NOTE: Ignoring %s which is a redirect to %s:' % (entry.title, code)) target = None @@ -279,6 +281,11 @@ wikipedia.output( u'Redirect target %s is not a redirect.' % secondRedir.aslink()) + except wikipedia.BadTitle, e: + # str(e) is in the format 'BadTitle: [[Foo]]' + wikipedia.output( + u'Redirect target %s is not a valid page title.' + % str(e)[10:]) except wikipedia.NoPage: wikipedia.output( u'Redirect target %s doesn\'t exist.'

1 0

[Pywikipedia-l] [ pywikipediabot-Bugs-1855071 ] "redirect.py double -xml:xx -namespace:x" crashing
by SourceForge.net 20 Dec '07

20 Dec '07

Bugs item #1855071, was opened at 2007-12-20 19:10 Message generated for change (Tracker Item Submitted) made by Item Submitter You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1855071&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None Status: Open Resolution: None Priority: 5 Private: No Submitted By: Nicolas Dumazet (nicdumz) Assigned to: Nobody/Anonymous (nobody) Summary: "redirect.py double -xml:xx -namespace:x" crashing Initial Comment: It is not even loading the XML, i get an instant error : python redirect.py double -namespace:0 -xml:frwiki-20071203-pages-articles.xml Checked for running processes. 1 processes currently running, including the current process. Reading XML dump... Traceback (most recent call last): File "redirect.py", line 377, in <module> main() File "redirect.py", line 373, in main bot.run() File "redirect.py", line 328, in run self.fix_double_redirects() File "redirect.py", line 255, in fix_double_redirects for redir_name in self.generator.retrieve_double_redirects(): File "redirect.py", line 199, in retrieve_double_redirects dict = self.get_redirects_from_dump() File "redirect.py", line 123, in get_redirects_from_dump if self.namespace and self.namespace != entry.namespace: AttributeError: XmlEntry instance has no attribute 'namespace' ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1855071&group_…

1 0

[Pywikipedia-l] [ pywikipediabot-Bugs-1855044 ] redirect.py crashes when finding a bad page
by SourceForge.net 20 Dec '07

20 Dec '07

Bugs item #1855044, was opened at 2007-12-20 18:20 Message generated for change (Tracker Item Submitted) made by Item Submitter You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1855044&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None Status: Open Resolution: None Priority: 5 Private: No Submitted By: Nicolas Dumazet (nicdumz) Assigned to: Nobody/Anonymous (nobody) Summary: redirect.py crashes when finding a bad page Initial Comment: When doing a simple "redirect.py double -xml:xx.xml", I got : Traceback (most recent call last): File "redirect.py", line 377, in <module> main() File "redirect.py", line 373, in main bot.run() File "redirect.py", line 328, in run self.fix_double_redirects() File "redirect.py", line 273, in fix_double_redirects secondTargetPage = secondRedir.getRedirectTarget() File "/home/nico/projets/pywikipedia/wikipedia.py", line 1576, in getRedirectTarget self.get() File "/home/nico/projets/pywikipedia/wikipedia.py", line 595, in get self._contents, self._isWatched, self.editRestriction = self._getEditPage(get_redirect = get_redirect, throttle = throttle, sysop = sysop, nofollow_redirects=nofollow_redirects) File "/home/nico/projets/pywikipedia/wikipedia.py", line 679, in _getEditPage raise BadTitle('BadTitle: %s' % self) wikipedia.BadTitle: BadTitle: [[../Projet/Sciences/Champs magnétiques B et H]] Now, the redirect page was : #REDIRECT [[../Projet/Sciences/Champs magnétiques B et H]], which is not correct. But I think that redirect.py is supposed to ask the user if not skipping it, instead of crashing ;) Thanks, Nicolas Dumazet. ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1855044&group_…

1 0

[Pywikipedia-l] SVN: [4740] trunk/pywikipedia/wikipedia.py
by russblau＠svn.wikimedia.org 20 Dec '07

20 Dec '07

Revision: 4740 Author: russblau Date: 2007-12-20 17:09:58 +0000 (Thu, 20 Dec 2007) Log Message: ----------- Add another replaceExcept exception for <ref></ref> tags, and remove duplicate comment Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-12-20 17:03:45 UTC (rev 4739) +++ trunk/pywikipedia/wikipedia.py 2007-12-20 17:09:58 UTC (rev 4740) @@ -2768,10 +2768,11 @@ 'noinclude': re.compile(r'(?is)<noinclude>.*?</noinclude>'), # wiki tags are ignored inside nowiki tags. 'nowiki': re.compile(r'(?is)<nowiki>.*?</nowiki>'), + # preformatted text + 'pre': re.compile(r'(?ism)<pre>.*?</pre>'), + # inline references + 'ref': re.compile(r'(?ism)<ref[ >].*?</ref>'), # lines that start with a space are shown in a monospace font and - # have whitespace preserved, with wiki tags being ignored. - 'pre': re.compile(r'(?is)<pre>.*?</pre>'), - # lines that start with a space are shown in a monospace font and # have whitespace preserved. 'startspace': re.compile(r'(?m)^ (.*?)$'), # tables often have whitespace that is used to improve wiki

1 0

[Pywikipedia-l] SVN: [4739] trunk/pywikipedia/pagegenerators.py
by rotem＠svn.wikimedia.org 20 Dec '07

20 Dec '07

Revision: 4739 Author: rotem Date: 2007-12-20 17:03:45 +0000 (Thu, 20 Dec 2007) Log Message: ----------- Spacing. Modified Paths: -------------- trunk/pywikipedia/pagegenerators.py Modified: trunk/pywikipedia/pagegenerators.py =================================================================== --- trunk/pywikipedia/pagegenerators.py 2007-12-20 07:19:29 UTC (rev 4738) +++ trunk/pywikipedia/pagegenerators.py 2007-12-20 17:03:45 UTC (rev 4739) @@ -123,7 +123,7 @@ for page in site.allpages(start=title, namespace=namespace, includeredirects = includeredirects): yield page -def PrefixingPageGenerator(prefix, namespace=None, includeredirects=True, site = None): +def PrefixingPageGenerator(prefix, namespace = None, includeredirects = True, site = None): for page in AllpagesPageGenerator(prefix, namespace, includeredirects, site): if page.titleWithoutNamespace().startswith(prefix): yield page @@ -292,7 +292,7 @@ break yield wikipedia.Page(site, title) -def LinksearchPageGenerator(link, step=500, site = None): +def LinksearchPageGenerator(link, step=500, site=None): """Yields all pages that include a specified link, according to [[Special:Linksearch]]. Retrieves in chunks of size "step" (default 500). @@ -503,7 +503,7 @@ current_year = date.formatYear(site.lang, i ) yield wikipedia.Page(site, current_year) -def DayPageGenerator(startMonth=1, endMonth=12, site=None): +def DayPageGenerator(startMonth = 1, endMonth = 12, site = None): if site is None: site = wikipedia.getSite() fd = date.FormatDate(site) @@ -647,7 +647,7 @@ wikipedia.output(unicode(e)) self.queue.put(None) # to signal end of list -def PreloadingGenerator(generator, pageNumber=60): +def PreloadingGenerator(generator, pageNumber = 60): """ Yields the same pages as generator generator. Retrieves 60 pages (or another number specified by pageNumber), loads them using

1 0

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

pywikibot