Pywikipedia-l November 2008

pywikipedia-l@lists.wikimedia.org

20 participants
178 discussions

SVN: [6096] trunk/pywikipedia/djvutext.py

by nicdumz＠svn.wikimedia.org

Revision: 6096 Author: nicdumz Date: 2008-11-14 06:55:36 +0000 (Fri, 14 Nov 2008) Log Message: ----------- identation fix + encoding of commands Modified Paths: -------------- trunk/pywikipedia/djvutext.py Modified: trunk/pywikipedia/djvutext.py =================================================================== --- trunk/pywikipedia/djvutext.py 2008-11-14 06:16:35 UTC (rev 6095) +++ trunk/pywikipedia/djvutext.py 2008-11-14 06:55:36 UTC (rev 6096) @@ -20,7 +20,7 @@ """ __version__ = '$Id$' import wikipedia -import os +import os, sys import config, codecs # This is required for the text that is shown when you run this script @@ -89,8 +89,8 @@ self.prefix = linkingPage.titleWithoutNamespace() if self.prefix[0:6] == 'Liber:': self.prefix = self.prefix[6:] - wikipedia.output(u"Using prefix %s" % self.prefix) - gen = self.PagesGenerator() + wikipedia.output(u"Using prefix %s" % self.prefix) + gen = self.PagesGenerator() site = wikipedia.getSite() self.username = config.usernames[site.family.name][site.lang] @@ -100,8 +100,8 @@ self.treat(pageno) def has_text(self): - cmd = "djvudump \"%s\" > \"%s\".out" % (self.djvu, self.djvu) - os.system ( cmd ) + cmd = u"djvudump \"%s\" > \"%s\".out" % (self.djvu, self.djvu) + os.system ( cmd.encode(sys.stdout.encoding) ) f = codecs.open("%s.out" % self.djvu, 'r', config.textfile_encoding, 'replace') @@ -112,7 +112,7 @@ def get_page(self, pageno): wikipedia.output(unicode("fetching page %d" % (pageno))) cmd = "djvutxt -page=%d \"%s\" \"%s.out\"" % (pageno, self.djvu, self.djvu) - os.system ( cmd ) + os.system ( cmd.encode(sys.stdout.encoding) ) f = codecs.open("%s.out" % self.djvu, 'r', config.textfile_encoding, 'replace')

15 years, 6 months

SVN: [6095] trunk/pywikipedia/djvutext.py

by nicdumz＠svn.wikimedia.org

Revision: 6095 Author: nicdumz Date: 2008-11-14 06:16:35 +0000 (Fri, 14 Nov 2008) Log Message: ----------- What about some big cleanups ? Make it unicode safe, remove tabs, fix indentation, ensure that debug mode can be activated as promised... And various style changes Modified Paths: -------------- trunk/pywikipedia/djvutext.py Modified: trunk/pywikipedia/djvutext.py =================================================================== --- trunk/pywikipedia/djvutext.py 2008-11-13 12:01:50 UTC (rev 6094) +++ trunk/pywikipedia/djvutext.py 2008-11-14 06:16:35 UTC (rev 6095) @@ -33,59 +33,53 @@ # NOTE: Put a good description here, and add translations, if possible! msg = { 'en': u'Robot: creating page with text extracted from DjVu', - 'ar': u'روبوت: إنشاء صفحة بنص مستخرج من DjVu', - 'fr': u'Bot: creating page with texte extracted from DjVu', + 'ar': u'روبوت: إنشاء صفحة بنص مستخرج من DjVu', + 'fr': u'Bot: creating page with texte extracted from DjVu', 'pt': u'Bot: criando página com texto extraído do DjVu', } # On English Wikisource, {{blank page}} is used to track blank pages. # It may be omitted by adding an empty string like has been done for 'fr'. blank = { 'en': u'{{blank page}}', - 'fr': u'', + 'fr': u'', 'pt': u'', } - def __init__(self, djvu, index, pages): + def __init__(self, djvu, index, pages, ask=False, debug=False): """ Constructor. Parameters: - djvu : filename - index : page name - pages : page range + djvu : filename + index : page name + pages : page range """ self.djvu = djvu self.index = index - self.pages = pages - self.debug = False - self.ask = False + self.pages = pages + self.debug = debug + self.ask = ask def NoOfImages(self): - cmd = "djvused -e 'n' \"%s\"" % (self.djvu) + cmd = "djvused -e 'n' \"%s\"" % (self.djvu) count = os.popen( cmd ).readline().rstrip() - #count = count[:-1] - print "page count = %s" % count - count = int(count) - print "page count = %d" % count - return int(count) + count = int(count) + wikipedia.output("page count = %d" % count) + return count def PagesGenerator(self): start = 1 - end = self.NoOfImages() + end = self.NoOfImages() - if self.pages: - pos = self.pages.find('-') - if pos != -1: - start = self.pages[:pos] - if pos < len(self.pages)-1: - end = self.pages[pos+1:] - end = int(end) - else: - start = self.pages - end = int(start) - i = int(start) - print "processing pages %d-%d" % (i, end) - while i <= end: - yield i - i=i+1 + if self.pages: + pos = self.pages.find('-') + if pos != -1: + start = int(self.pages[:pos]) + if pos < len(self.pages)-1: + end = int(self.pages[pos+1:]) + else: + start = int(self.pages) + end = start + wikipedia.output(u"Processing pages %d-%d" % (start, end)) + return range(start, end+1) def run(self): # Set the edit summary message @@ -93,16 +87,16 @@ linkingPage = wikipedia.Page(wikipedia.getSite(), self.index) self.prefix = linkingPage.titleWithoutNamespace() - if self.prefix[0:6] == 'Liber:': - self.prefix = self.prefix[6:] - wikipedia.output("Using prefix %s" % self.prefix) - gen = self.PagesGenerator() - - site = wikipedia.getSite() + if self.prefix[0:6] == 'Liber:': + self.prefix = self.prefix[6:] + wikipedia.output(u"Using prefix %s" % self.prefix) + gen = self.PagesGenerator() + + site = wikipedia.getSite() self.username = config.usernames[site.family.name][site.lang] for pageno in gen: - wikipedia.output("Processing page %d" % pageno) + wikipedia.output("Processing page %d" % pageno) self.treat(pageno) def has_text(self): @@ -113,14 +107,7 @@ s = f.read() f.close() - - import string - blah = string.find(s, 'TXTz') # text layers are described with this value - - if string.find(s, 'TXTz') >= 0: - return True - else: - return False + return s.find('TXTz') >= 0 def get_page(self, pageno): wikipedia.output(unicode("fetching page %d" % (pageno))) @@ -137,54 +124,57 @@ """ Loads the given page, does some changes, and saves it. """ - site = wikipedia.getSite() - page_namespace = site.family.namespaces[104][site.lang] - page = wikipedia.Page(site, '%s:%s/%d' % (page_namespace, self.prefix, pageno) ) - exists = page.exists() + site = wikipedia.getSite() + page_namespace = site.family.namespaces[104][site.lang] + page = wikipedia.Page(site, u'%s:%s/%d' % (page_namespace, self.prefix, pageno) ) + exists = page.exists() djvutxt = self.get_page(pageno) if not djvutxt: - djvutxt = wikipedia.translate(wikipedia.getSite(), self.blank) - text = '<noinclude>{{PageQuality|1|%s}}<div class="pagetext">%s</noinclude>%s<noinclude><references/></div></noinclude>' % (self.username,"\n\n\n",djvutxt) + djvutxt = wikipedia.translate(wikipedia.getSite(), self.blank) + text = u'<noinclude>{{PageQuality|1|%s}}<div class="pagetext">\n\n\n</noinclude>%s<noinclude><references/></div></noinclude>' % (self.username,djvutxt) # convert to wikisyntax # this adds a second line feed, which makes a new paragraph - text = text.replace('', "\n") + text = text.replace('', "\n") # only save if something was changed # automatically ask if overwriting an existing page - old_text = '' + ask = self.ask if exists: - ask = 'y' + ask = True old_text = page.get() + if old_text == text: + wikipedia.output(u"No changes were needed on %s" % page.aslink()) + return + else: + old_text = '' - if not exists or text != old_text: - # Show the title of the page we're working on. - # Highlight the title in purple. - wikipedia.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" % page.title()) - # show what was changed - if exists: - wikipedia.showDiff(old_text, text) - else: - wikipedia.output(text) - if not self.debug: - choice = 'y' - if ask: - choice = wikipedia.inputChoice(u'Do you want to accept these changes?', ['Yes', 'No'], ['y', 'N'], 'N') - if choice == 'y': - try: - # Save the page - page.put_async(text) - except wikipedia.LockedPage: - wikipedia.output(u"Page %s is locked; skipping." % page.aslink()) - except wikipedia.EditConflict: - wikipedia.output(u'Skipping %s because of edit conflict' % (page.title())) - except wikipedia.SpamfilterError, error: - wikipedia.output(u'Cannot change %s because of spam blacklist entry %s' % (page.title(), error.url)) + wikipedia.output(u"\n\n>>> \03{lightpurple}%s\03{default} <<<" % page.title()) + wikipedia.showDiff(old_text, text) + if self.debug: + wikipedia.inputChoice(u'Debug mode... Press enter to continue', [], [], 'dummy') + return + if ask: + choice = wikipedia.inputChoice(u'Do you want to accept these changes?', ['Yes', 'No'], ['y', 'N'], 'N') + else: + choice = 'y' + if choice == 'y': + try: + # Save the page + page.put_async(text) + except wikipedia.LockedPage: + wikipedia.output(u"Page %s is locked; skipping." % page.aslink()) + except wikipedia.EditConflict: + wikipedia.output(u'Skipping %s because of edit conflict' % (page.title())) + except wikipedia.SpamfilterError, error: + wikipedia.output(u'Cannot change %s because of spam blacklist entry %s' % (page.title(), error.url)) + + def main(): import os index = None @@ -199,15 +189,15 @@ if arg.startswith("-debug"): debug = True elif arg.startswith("-ask"): - ask = True + ask = True elif arg.startswith("-djvu:"): - djvu = arg[6:] - elif arg.startswith("-index:"): - index = arg[7:] + djvu = arg[6:] + elif arg.startswith("-index:"): + index = arg[7:] elif arg.startswith("-pages:"): - pages = arg[7:] + pages = arg[7:] else: - print "Unknown argument %s" % arg + wikipedia.output(u"Unknown argument %s" % arg) # Check the djvu file exists if djvu: @@ -222,7 +212,7 @@ index_page = wikipedia.Page(site, index) if site.family.name != 'wikisource': - raise wikipedia.PageNotFound(u"Found family '%s'; Wikisource required." % site.family.name) + raise wikipedia.PageNotFound(u"Found family '%s'; Wikisource required." % site.family.name) if not index_page.exists() and index_page.namespace() == 0: index_namespace = wikipedia.Page(site, 'MediaWiki:Proofreadpage index namespace').get() @@ -231,15 +221,14 @@ u"%s:%s" % (index_namespace, index)) if not index_page.exists(): - raise wikipedia.NoPage("Page '%s' does not exist" % index) + raise wikipedia.NoPage(u"Page '%s' does not exist" % index) - wikipedia.output(u"uploading text from %s to %s" % (djvu, index_page) ) + wikipedia.output(u"uploading text from %s to %s" % (djvu, index_page.aslink()) ) - bot = DjVuTextBot(djvu, index, pages) - if not bot.has_text(): + bot = DjVuTextBot(djvu, index, pages, ask, debug) + if not bot.has_text(): raise ValueError("No text layer in djvu file") - bot.ask = ask bot.run() else: wikipedia.showHelp()

15 years, 6 months

[ pywikipediabot-Bugs-2269688 ] Unicode error with djvutext.py

by SourceForge.net

Bugs item #2269688, was opened at 2008-11-12 15:03 Message generated for change (Comment added) made by yannforget You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=2269688&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None >Status: Open Resolution: Fixed Priority: 5 Private: No Submitted By: Yann Forget (yannforget) Assigned to: Nobody/Anonymous (nobody) Summary: Unicode error with djvutext.py Initial Comment: on fr.wikisource: python djvutext.py -index:Livre:Le_Th%C3%A9%C3%A2tre_de_la_R%C3%A9volution._Le_Quatorze_Juillet._Danton._Les_Loups.djvu -djvu:Le_quatorze_juillet_Danton_Les_loups.djvu -pages:375 Checked for running processes. 1 processes currently running, including the current process. Traceback (most recent call last): File "djvutext.py", line 249, in <module> main() File "djvutext.py", line 236, in main wikipedia.output("uploading text from %s to %s" % (djvu, index_page) ) UnicodeDecodeError: 'ascii' codec can't decode byte 0xc3 in position 13: ordinal not in range(128) python version.py Pywikipedia [http] trunk/pywikipedia (r6084, Nov 11 2008, 21:51:31) Python 2.5.2 (r252, Sep 13 2008, 22:55:01) [GCC 4.1.2 (Gentoo 4.1.2 p1.1)] ---------------------------------------------------------------------- >Comment By: Yann Forget (yannforget) Date: 2008-11-13 20:42 Message: I still have the same error. Yann ---------------------------------------------------------------------- Comment By: NicDumZ — Nicolas Dumazet (nicdumz) Date: 2008-11-12 18:41 Message: Au loup !! This display error should be fixed by r6089. Please re-comment on this bug if it is not the case :) ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=2269688&group_…

15 years, 6 months

[ pywikipediabot-Feature Requests-1970099 ] Provide (user-)config.py variable for disabling output()

by SourceForge.net

Feature Requests item #1970099, was opened at 2008-05-23 06:31 Message generated for change (Comment added) made by nicdumz You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603141&aid=1970099&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None Status: Open Priority: 5 Private: No Submitted By: Melancholie (melancholie) Assigned to: Nobody/Anonymous (nobody) Summary: Provide (user-)config.py variable for disabling output() Initial Comment: If you run a bot in autonomous mode (interwiki.py), you actually do not have to have a look on the console output; rather on the "wiki" end result. Printing out so many stuff is just slowing down the autonomous script process. It would be nice if there was a variable defined in config.py (for user-config.py) that would allow to not get all this stuff printed all the time by setting the var = True/False. See function output() ---------------------------------------------------------------------- Comment By: NicDumZ — Nicolas Dumazet (nicdumz) Date: 2008-11-13 13:07 Message: It's already available. See set_debug in __init__.py (rewrite) ---------------------------------------------------------------------- Comment By: Multichill (multichill) Date: 2008-11-13 12:51 Message: We could implement log levels in pywikipedia. For example 7 levels, 7 = debug, 6 = informational, .... 1 = crash. Might be something to take into account for the rewrite. ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603141&aid=1970099&group_…

15 years, 6 months

[ pywikipediabot-Bugs-2082329 ] movepages.py with -del does not delete talk page redirect

by SourceForge.net

Bugs item #2082329, was opened at 2008-08-29 14:11 Message generated for change (Comment added) made by multichill You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=2082329&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None Status: Open Resolution: None Priority: 5 Private: No Submitted By: Uberfuzzy (uberfuzzy) Assigned to: Nobody/Anonymous (nobody) Summary: movepages.py with -del does not delete talk page redirect Initial Comment: Pywikipedia [http] trunk/pywikipedia (r5858, Aug 28 2008, 11:35:51) Python 2.5.1 (r251:54863, Apr 18 2007, 08:51:08) [MSC v.1310 32 bit (Intel)] when using the movepages.py script with the -del option to delete the leftover redirect, only the redirect for the page is deleted, but not the talk page redirect. ---------------------------------------------------------------------- >Comment By: Multichill (multichill) Date: 2008-11-13 13:06 Message: Mediawiki now supports moving pages without leaving redirects for bots. Maybe the -del option should just move the page with this option enabled, save you the deleting of a useless page. ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=2082329&group_…

15 years, 6 months

[ pywikipediabot-Bugs-2235697 ] family.py, localised REDIRECT version for hr

by SourceForge.net

Bugs item #2235697, was opened at 2008-11-07 19:21 Message generated for change (Comment added) made by multichill You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=2235697&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None >Status: Closed Resolution: None Priority: 5 Private: No Submitted By: Nobody/Anonymous (nobody) >Assigned to: Multichill (multichill) Summary: family.py, localised REDIRECT version for hr Initial Comment: Please add localised redirect instruction for hr to 'redirect' section in family.py 'hr': [u'REDIRECT', u'PREUSMJERI'], Thanks. Pywikipedia [http] trunk/pywikipedia (r6073, Nov 06 2008, 20:53:12) Python 2.5.1 (r251:54863, Apr 18 2007, 08:51:08) [MSC v.1310 32 bit (Intel)] ---------------------------------------------------------------------- >Comment By: Multichill (multichill) Date: 2008-11-13 13:02 Message: Added in svn revision 6094 ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=2235697&group_…

15 years, 6 months

SVN: [6094] trunk/pywikipedia/family.py

by multichill＠svn.wikimedia.org

Revision: 6094 Author: multichill Date: 2008-11-13 12:01:50 +0000 (Thu, 13 Nov 2008) Log Message: ----------- Localised redirect template for hr, see bug #2235697 : 'hr': [u'PREUSMJERI'], Modified Paths: -------------- trunk/pywikipedia/family.py Modified: trunk/pywikipedia/family.py =================================================================== --- trunk/pywikipedia/family.py 2008-11-12 20:32:26 UTC (rev 6093) +++ trunk/pywikipedia/family.py 2008-11-13 12:01:50 UTC (rev 6094) @@ -2925,6 +2925,7 @@ 'fi': [u'ohjaus', u'uudelleenohjaus'], 'ga': [u'athsheoladh'], 'he': [u'הפניה'], + 'hr': [u'PREUSMJERI'], 'hu': [u'átirányítás'], 'id': [u'alih'], 'is': [u'tilvísun'],

15 years, 6 months

[ pywikipediabot-Feature Requests-1939195 ] Translation for simple wiki

by SourceForge.net

Feature Requests item #1939195, was opened at 2008-04-10 10:01 Message generated for change (Comment added) made by multichill You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603141&aid=1939195&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: Interface Improvements Group: None >Status: Closed Priority: 5 Private: No Submitted By: Nobody/Anonymous (nobody) Assigned to: Nobody/Anonymous (nobody) Summary: Translation for simple wiki Initial Comment: Please add the language entries for Simple English Wikipedia into the scripts. It can be annoying re-entering them after checking out the files. I am Chenzw on Wikipedia. ---------------------------------------------------------------------- >Comment By: Multichill (multichill) Date: 2008-11-13 12:53 Message: Which language entries? Nothing attached. Closing this request. ---------------------------------------------------------------------- Comment By: Nobody/Anonymous (nobody) Date: 2008-06-23 22:25 Message: Logged In: NO close this? ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603141&aid=1939195&group_…

15 years, 6 months

[ pywikipediabot-Feature Requests-1970099 ] Provide (user-)config.py variable for disabling output()

by SourceForge.net

Feature Requests item #1970099, was opened at 2008-05-23 06:31 Message generated for change (Comment added) made by multichill You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603141&aid=1970099&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None Status: Open Priority: 5 Private: No Submitted By: Melancholie (melancholie) Assigned to: Nobody/Anonymous (nobody) Summary: Provide (user-)config.py variable for disabling output() Initial Comment: If you run a bot in autonomous mode (interwiki.py), you actually do not have to have a look on the console output; rather on the "wiki" end result. Printing out so many stuff is just slowing down the autonomous script process. It would be nice if there was a variable defined in config.py (for user-config.py) that would allow to not get all this stuff printed all the time by setting the var = True/False. See function output() ---------------------------------------------------------------------- >Comment By: Multichill (multichill) Date: 2008-11-13 12:51 Message: We could implement log levels in pywikipedia. For example 7 levels, 7 = debug, 6 = informational, .... 1 = crash. Might be something to take into account for the rewrite. ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603141&aid=1970099&group_…

15 years, 6 months

[ pywikipediabot-Feature Requests-1910658 ] Change links pointing to redirect pages

by SourceForge.net

Feature Requests item #1910658, was opened at 2008-03-09 22:00 Message generated for change (Comment added) made by multichill You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603141&aid=1910658&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None >Status: Closed Priority: 5 Private: No Submitted By: Nobody/Anonymous (nobody) Assigned to: Nobody/Anonymous (nobody) Summary: Change links pointing to redirect pages Initial Comment: Hello, I'm admin on mediawiki site on my companie's intranet. I have renamed a lot of pages to integrate them in a new namespace. As a result i have a lot of links that link to the olds names of the pages which are became redirect pages. That's a problem for us and for cosmetic reason i have to update all this links to link directly on the renamed pages. I dont have found a pywikipedia script who can do this ? Perhaps the idea is to collect all the redirect page like pywikepedia do it for the double redirect ones and scan all the pages in the wiki to see if they point to one of this redirect page to finally change this links in consequence.. Thanks for your help Best regards ---------------------------------------------------------------------- >Comment By: Multichill (multichill) Date: 2008-11-13 12:49 Message: What leogregianin said : fixing_redirects.py is what you need. I'm closing this request. ---------------------------------------------------------------------- Comment By: Leonardo Gregianin (leogregianin) Date: 2008-07-19 04:59 Message: Logged In: YES user_id=1136737 Originator: NO fixing_redirects.py can you helped ---------------------------------------------------------------------- Comment By: AndreasJS (andreasjs) Date: 2008-03-10 22:19 Message: Logged In: YES user_id=1738850 Originator: NO I made a program that does this. (Attached) Please try it out to see if it does what you want. File Added: resolveredirects.py ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603141&aid=1910658&group_…

15 years, 6 months

← Newer
1
...
7
8
9
10
11
12
13
...
18
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

Pywikipedia-l November 2008