Pywikipedia-l November 2007

pywikipedia-l@lists.wikimedia.org

29 participants
285 discussions

SVN: [4516] trunk/pywikipedia/daemonize.py
by btongminh＠svn.wikimedia.org 08 Nov '07

08 Nov '07

Revision: 4516 Author: btongminh Date: 2007-11-08 13:14:24 +0000 (Thu, 08 Nov 2007) Log Message: ----------- Fix log opening Modified Paths: -------------- trunk/pywikipedia/daemonize.py Modified: trunk/pywikipedia/daemonize.py =================================================================== --- trunk/pywikipedia/daemonize.py 2007-11-08 12:56:01 UTC (rev 4515) +++ trunk/pywikipedia/daemonize.py 2007-11-08 13:14:24 UTC (rev 4516) @@ -33,7 +33,7 @@ os.close(2) os.open('/dev/null', os.O_RDWR) if redirect_std: - os.open(redirect_std, os.O_RDONLY | os.O_APPEND) + os.open(redirect_std, os.O_WRONLY | os.O_APPEND) else: os.dup2(0, 1) os.dup2(1, 2)

1 0

SVN: [4515] trunk/pywikipedia
by btongminh＠svn.wikimedia.org 08 Nov '07

08 Nov '07

Revision: 4515 Author: btongminh Date: 2007-11-08 12:56:01 +0000 (Thu, 08 Nov 2007) Log Message: ----------- Add daemonize code Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Added Paths: ----------- trunk/pywikipedia/daemonize.py Added: trunk/pywikipedia/daemonize.py =================================================================== --- trunk/pywikipedia/daemonize.py (rev 0) +++ trunk/pywikipedia/daemonize.py 2007-11-08 12:56:01 UTC (rev 4515) @@ -0,0 +1,52 @@ +import sys, os + +__version__ = '$Id: $' + +is_daemon = False + +def daemonize(close_fd = True, chdir = True, write_pid = False, redirect_std = None): + """ Daemonize the current process. Only works on POSIX compatible operating + systems. The process will fork to the background and return control to + the terminal. + + Arguments: + - close_fd: Close the standard streams and replace them by /dev/null + - chdir: Change the current working directory to / + - write_pid: Write the pid to sys.argv[0] + '.pid' + - redirect_std: Filename to redirect stdout and stdin to + """ + + # Fork away + if not os.fork(): + # Become session leader + os.setsid() + # Fork again to prevent the process from acquiring a + # controlling terminal + pid = os.fork() + if not pid: + global is_daemon + is_daemon = True + + if close_fd: + os.close(0) + os.close(1) + os.close(2) + os.open('/dev/null', os.O_RDWR) + if redirect_std: + os.open(redirect_std, os.O_RDONLY | os.O_APPEND) + else: + os.dup2(0, 1) + os.dup2(1, 2) + if chdir: + os.chdir('/') + return + else: + # Write out the pid + f = open(sys.argv[0] + '.pid', 'w') + f.write(str(pid)) + f.close() + os._exit(0) + else: + # Exit to return control to the terminal + # os._exit to prevent the cleanup to run + os._exit(0) \ No newline at end of file Property changes on: trunk/pywikipedia/daemonize.py ___________________________________________________________________ Name: svn:eol-style + native Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-11-08 12:50:25 UTC (rev 4514) +++ trunk/pywikipedia/wikipedia.py 2007-11-08 12:56:01 UTC (rev 4515) @@ -5000,6 +5000,12 @@ output(u'Pywikipediabot %s' % (version.getversion())) output(u'Python %s' % (sys.version)) verbose += 1 + elif arg == '-daemonize': + import daemonize + daemonize.daemonize() + elif arg.startswith('-daemonize:'): + import daemonize + daemonize.daemonize(redirect_std = arg[11:]) else: # the argument is not global. Let the specific bot script care # about it. @@ -5377,6 +5383,10 @@ wikipedia, wiktionary, wikitravel, ... This will override the configuration in user-config.py. +-daemonize:xyz Immediately returns control to the terminal and redirects + stdout and stderr to xyz (only use for bots that require + no input from stdin). + -help Shows this help text. -log Enable the logfile. Logs will be stored in the logs

1 0

SVN: [4514] trunk/pywikipedia/commonsdelinker/checkusage.py
by btongminh＠svn.wikimedia.org 08 Nov '07

08 Nov '07

Revision: 4514 Author: btongminh Date: 2007-11-08 12:50:25 +0000 (Thu, 08 Nov 2007) Log Message: ----------- white space fixes Modified Paths: -------------- trunk/pywikipedia/commonsdelinker/checkusage.py Modified: trunk/pywikipedia/commonsdelinker/checkusage.py =================================================================== --- trunk/pywikipedia/commonsdelinker/checkusage.py 2007-11-07 20:50:52 UTC (rev 4513) +++ trunk/pywikipedia/commonsdelinker/checkusage.py 2007-11-08 12:50:25 UTC (rev 4514) @@ -91,19 +91,19 @@ self._conn = httplib.HTTPConnection(host) #self._conn.set_debuglevel(100) self._conn.connect() - + def request(self, method, path, headers, data): if not headers: headers = {} if not data: data = '' headers['Connection'] = 'Keep-Alive' headers['User-Agent'] = 'MwClient/' + __ver__ - + try: self._conn.request(method, path, data, headers) except socket.error, e: self._conn.close() raise - + try: res = self._conn.getresponse() except httplib.BadStatusLine: @@ -111,16 +111,16 @@ self._conn.connect() self._conn.request(method, path, data, headers) res = self._conn.getresponse() - + if res.status >= 500: self._conn.request(method, path, data, headers) res = self._conn.getresponse() - + if res.status != 200: raise RuntimeError, (res.status, res) - + return res - + def query_api(self, host, path, **kwargs): data = urlencode([(k, v.encode('utf-8')) for k, v in kwargs.iteritems()]) if path.endswith('query.php'): @@ -132,7 +132,7 @@ method = 'POST' else: raise ValueError('Unknown api %s' % repr(api)) - + try: res = self.request(method, query_string, {'Host': host, 'Content-Type': 'application/x-www-form-urlencoded'}, data) @@ -151,7 +151,7 @@ return data def close(self): self._conn.close() - + class HTTPPool(list): def __init__(self, retry_timeout = 10, max_retries = -1, callback = lambda *args: None): @@ -211,7 +211,7 @@ conn.close() del self[:] - + class CheckUsage(object): def __init__(self, limit = 100, mysql_default_server = 3, mysql_host_prefix = 'sql-s', mysql_kwargs = {}, @@ -229,7 +229,7 @@ self.http_callback = http_callback if no_db: return - + self.mysql_host_prefix = mysql_host_prefix if 'host' in mysql_kwargs: del mysql_kwargs['host'] self.mysql_kwargs = mysql_kwargs @@ -252,10 +252,10 @@ self.unknown_families = [] # Mapping family name -> family object self.known_families = {} - + database, cursor = self.connect_mysql(mysql_host_prefix + str(mysql_default_server)) self.servers[mysql_default_server] = (database, cursor) - + # Find where the databases are located cursor.execute('SELECT dbname, domain, server FROM toolserver.wiki ORDER BY size DESC LIMIT %s', (limit, )) for dbname, domain, server in cursor.fetchall(): @@ -275,8 +275,8 @@ self.databases[dbname] = self.servers[server] self.domains[dbname] = domain - - + + def connect_mysql(self, host): # A bug in MySQLdb 1.2.1_p will force you to set # all your connections to use_unicode = False. @@ -304,7 +304,7 @@ usage = self.get_usage_db(dbname, image, True) for link in usage: yield self.sites[dbname], link - + def get_usage_db(self, dbname, image, shared = False): #image = strip_image(image) lang, family_name = self.sites[dbname] @@ -325,7 +325,7 @@ else: title = stripped_title yield page_namespace, stripped_title, title - + def get_usage_live(self, site, image, shared = False): self.connect_http()

1 0

SVN: [4513] trunk/pywikipedia/copyright.py
by valhallasw＠svn.wikimedia.org 07 Nov '07

07 Nov '07

Revision: 4513 Author: valhallasw Date: 2007-11-07 20:50:52 +0000 (Wed, 07 Nov 2007) Log Message: ----------- Fixed forgotten wikipedia.makepath Modified Paths: -------------- trunk/pywikipedia/copyright.py Modified: trunk/pywikipedia/copyright.py =================================================================== --- trunk/pywikipedia/copyright.py 2007-11-06 21:51:20 UTC (rev 4512) +++ trunk/pywikipedia/copyright.py 2007-11-07 20:50:52 UTC (rev 4513) @@ -298,7 +298,7 @@ def exclusion_file_list(): for i in pages_for_exclusion_database: path = wikipedia.config.datafilepath(appdir, i[0], i[2]) - wikipedia.makepath(path) + wikipedia.config.makepath(path) p = wikipedia.Page(wikipedia.getSite(i[0]), i[1]) yield p, path

1 0

Re: [Pywikipedia-l] Rewrite thoughts (includes replies to the 4507-thread)
by Merlijn van Deen 07 Nov '07

07 Nov '07

Hello all, @Misza: > Ok now. As I am a complete lamer when it comes to team coding, coding standards etc., I'll take your words for it, but have nonetheless several > lame questions. Your comments are very valuable to me, actually. And lame questions don't exist ;) @Misza > If team coding is a democracy then I vote for > camelCaseWithFirstWordLowercase. > It's a strong standard in Java (which is a close cousin to Python) and is > frequent in Python too (take Twisted for an example). @Russell > Misza will be disappointed; PEP 8 says to use lowercase with underscores for function and method names. ;) I have to say I like getInfoFromBlah better than get_info_from_blah.. maybe it's just because the underscore is relatively hard to type ;). For classes we should keep the CamelCaseWithFirstWordUppercase style mentioned in the PEP. @Misza: > Fine. I suggest the most common 1 indentation level = 4 spaces. I like 4 spaces, even though I know some people like true tabs better ;) >> * UTF8 encoding for all files (and using u'' for all strings), with UTF8 >> BOM > I'll assume gvim does that for me. o_O If you tell it to, I think it will. The BOM is just three characters in front of the text that define the text is in UTF8 format. (BOM stands for Byte Order Marker, which is necessary for UTF-16/32, where the byte order read depends on the system) @Misza >> One module per class > Java style again? ;) Not directly inspired by Java, but apparently Java does it this way, too :) > A module corresponds to one .py file (like http.py or api.py from our example) and a folder of these (data/) is a package, not module? Despite of this: >>>> import data >>>> data > <module 'data' from 'data/__init__.py'> Technically, that's the module 'data/__init__.py' of the package data, I suppose :) @Russell >> One module per class > I'd make an exception to this one to allow including subclasses in the same module, such as Page, ImagePage, and Category. @Rob > One module per class is a bit too much of a javaism for me too. It is not really efficient since it requires quite a lot of "import"s in the code, and actually finding a class becomes quite expensive! On this point, I agree with Russell. The current layout is Site and Page objects in wikipedia.py and Category in catlib.py. Having all subclasses of Page in Page.py (or something like that) sounds right to me. @Misza: >> Because adding structure afterwards is much harder, I think we first should decide on what modules/classes we want, then defining what functions we want and what these functions should do. After that, we can >> start writing unit tests and functions. > Where do we start? Edit http://www.botwiki.sno.cc/wiki/Rewrite and add classes until someone says enough? Well, something like that, yes. We do not need to have all functions defined in complete detail to start at the framework, but we do need to have a general setup for modules/packages and how they will interconnect. Bryan has some very good points about this, which I will address further on in this mail. >> This is not a very important point, but it's kinda interesting. With the >> new framework, it's easier to restructure existing functions, making translations easier. > Ok now, can you elaborate on this? I don't think I'm getting the point - we already same i18n - bots have localized edit summaries, the framework knows #REDIRECT and namespace names locales as well. > Are you talking about some whole new level of i18n? Interface i18n: 'Wilt U deze wijziging doorvoeren (J/N)' instead of 'Do you want to save this edit (Y/N)' ;) @Rob: >> On unit testing -- it may be difficult to write unit tests for methods that access the wiki, because the value returned will depend on the contents of the wiki at any given time. Maybe if we have a dedicated test wiki with at least some pages that are locked, so that they give predictable values, that would be a way around the problem. > You are talking about testing, not unit testing. For unit testing you will not need a sensibly filled wiki. The code that needs sensible data will be tested separately from the wiki-fetching code. Each piece of separable functionality should be tested separately with true unit tests. Let me check if I understand unit testing then: the only functions that will need unit testing with a 'live test wiki' would be the functions that either put or read from the wiki, correct? Any other functions that do processing should just use a test text; i.e. externalLinks() should read from a test text saved and return the links therein? @Bryan: > What is very important is that we clearly separate the different > layers that a framework consists of and get rid of functions like > replaceExceptInWhatever in the main module. In my opinion a proper > framework consists of three separate layers: > * High-level > * Middleware > * Lowware or core Sounds like a good starting point for adding structure. > The core functionality should consists of methods to get and put raw > page data, such as Page.get, Page.categories, Site.recentchanges, etc. > The middleware consists of commonly used functions such as replaceIn, > replaceImage, replaceCategory. The high-level software is the bot > itself. It performs tasks by calling the functions of the middle ware > and core. I'm not sure whether bots should be part of the *framework* in the first place. In my opinion, the framework should be installable through easy-install for example- in a central location in any case. This means that the framework resides in /usr/lib/python2.5/site-packages/ while the bots reside in /home/valhallasw/bots (or sth). > Related to this, the question of i18n. I strongly believe that > low-ware should never output things to stdout or ask something from > stdin. Yes and no. I think there should be a core module that handles the output in a structured manner, but it should not be used in other core modules. I.e. core module 'comm layer' should not use core module 'stdout'. I do agree that i18n is a higher level function than the pure output, and hence should be 'middleware'. > The core itself can also be divided into sublayers: > * Python equivalents of the functions that the API provides > * Abstract Page/List objects > * Generic API function > * Generic communication layer > (...) I think this is pretty much what we indeed need. I was also thinking if it would not be easier to just generate Page objects from an API query used in the python source. For example: http://en.wikipedia.org/w/api.php?action=query&prop=links&titles=Main%20Page would generate a Page 'Main Page' with only the linkedpages loaded, while http://en.wikipedia.org/w/api.php?action=query&generator=templates&titles=M… would generate a list of Page objects that would be very suitable for interwiki.py this does mean we will need a more sophisticated way of finding what data is already loaded and what data is not. Maybe we should use __getattr__ to more logically shape the class? Some nice warnings about non-preloaded data being used would be neat, too :) > What is important to consider is where the error correction is put. > Some errors are recoverable after retry. The lower in level such a > retry is placed, the less code duplication is required. However, retry > code placed to low may cause to catch to generic errors. 'make sure you only catch errors you expect' :). In general, we should only catch errors that the user does not need to hear about. This means that even retry-after errors might need to traverse to the upper layer so the user at least gets a message 'Server load too high, retrying in ...'. If there is a way to get this information back to the user without backtracking completely, I'd like to hear :) > Also the use of persistent HTTP connections makes the framework less > fool proof. Persistent HTTP connections makes an object that uses them > automatically unsuitable for sharing between threads. Is that so? A quick sketchup: page.get(): lock = threading.Lock() #thread safe lock lock.acquire() page_get_queue.append((self, query, lock)) lock.acquire(1) with the page getter releasing the lock. But I am probably ignoring some important thread safety now :) Then, last but not least: @Russell: > This raises an important question. Do we want to continue trying to > support every MediaWiki installation, including those that haven't > upgraded to more recent versions of MW? There are a number of wikis > in the families/ directory now that don't have any API support at all. PHP4 support will be dropped in december. All sites should update to PHP5 and then should upgrade to mediawiki 1.10+. And as Bryan described: > Branching is the key. The trunk should always be in sync with the > trunk of MediaWiki. Additionally each time a MediaWiki version is > released, the trunk should be forked into a compatible branch. I think we should start working with the current SVN release (or rather the version used on wikimedia sites), and branch when a new stable mediawiki version is released. Bug fixes can be made, but new features will only be in trunk, and we don't have to guarantee the trunk is compatible with older versions of mediawiki. > Personally, I only use my bots on Wikimedia Foundation sites, so it's not > an issue for me. But it may be for others. If we decide to go API-only, > then users of the non-current MediaWiki installations will have to use the > "old" pywikipediabot. So be it. The old framework works, it's just not as efficient and neat as the new one will be ;) -- Merlijn

5 4

[ pywikipediabot-Bugs-1827590 ] featured.py add empty lines
by SourceForge.net 07 Nov '07

07 Nov '07

Bugs item #1827590, was opened at 2007-11-07 16:09 Message generated for change (Tracker Item Submitted) made by Item Submitter You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1827590&group_… Please note that this message will contain a full copy of the comment thread, including the initial issue submission, for this request, not just the latest update. Category: None Group: None Status: Open Resolution: None Priority: 5 Private: No Submitted By: Warddr (warddr) Assigned to: Nobody/Anonymous (nobody) Summary: featured.py add empty lines Initial Comment: The bot always add empty lines in between two linkFA templates. example: http://nl.wikipedia.org/w/index.php?title=Schaken&diff=10073293&oldid=10070… Thanks ---------------------------------------------------------------------- You can respond by visiting: https://sourceforge.net/tracker/?func=detail&atid=603138&aid=1827590&group_…

1 0

Re: [Pywikipedia-l] Rewrite thoughts (includes replies to the 4507-thread)
by Bryan Tong Minh 07 Nov '07

07 Nov '07

On Nov 6, 2007 9:03 PM, Huji <huji.huji(a)gmail.com> wrote: > Moving to the API is great, but it has its limitations. For example, to my > knowledge, it is impossible to get a list of more than 500 pages from the > API, so perhaps we need to find a way to get a bigger list (for example for > autonomous tasks). > That's why you need to implement a generic List object that handles the query-continue stuff (see http://fisheye.ts.wikimedia.org/browse/bryan/mwclient/trunk/log.py?r=HEAD) Bryan

1 0

SVN: [4512] trunk/pywikipedia
by russblau＠svn.wikimedia.org 06 Nov '07

06 Nov '07

Revision: 4512 Author: russblau Date: 2007-11-06 21:51:20 +0000 (Tue, 06 Nov 2007) Log Message: ----------- Moved functions that depend on user's local configuration to the config.py module Modified Paths: -------------- trunk/pywikipedia/casechecker.py trunk/pywikipedia/category.py trunk/pywikipedia/config.py trunk/pywikipedia/copyright.py trunk/pywikipedia/imagecopy.py trunk/pywikipedia/interwiki.py trunk/pywikipedia/login.py trunk/pywikipedia/makecat.py trunk/pywikipedia/solve_disambiguation.py trunk/pywikipedia/spellcheck.py trunk/pywikipedia/splitwarning.py trunk/pywikipedia/userinterfaces/terminal_interface.py trunk/pywikipedia/warnfile.py trunk/pywikipedia/watchlist.py trunk/pywikipedia/weblinkchecker.py trunk/pywikipedia/welcome.py trunk/pywikipedia/wikipedia.py trunk/pywikipedia/wikipediatools.py Modified: trunk/pywikipedia/casechecker.py =================================================================== --- trunk/pywikipedia/casechecker.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/casechecker.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -153,7 +153,7 @@ raise ValueError(u'Suspects must be the same size') if not os.path.isabs(self.wikilogfile): - self.wikilogfile = wikipedia.datafilepath(self.wikilogfile) + self.wikilogfile = wikipedia.config.datafilepath(self.wikilogfile) try: self.wikilog = codecs.open(self.wikilogfile, 'a', 'utf-8') except IOError: Modified: trunk/pywikipedia/category.py =================================================================== --- trunk/pywikipedia/category.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/category.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -163,10 +163,10 @@ else: try: if not os.path.isabs(filename): - filename = wikipedia.datafilepath(filename) + filename = wikipedia.config.datafilepath(filename) f = bz2.BZ2File(filename, 'r') wikipedia.output(u'Reading dump from %s' - % wikipedia.shortpath(filename)) + % wikipedia.config.shortpath(filename)) databases = pickle.load(f) f.close() # keys are categories, values are 2-tuples with lists as entries. @@ -230,9 +230,9 @@ Saves the contents of the dictionaries superclassDB and catContentDB to disk. ''' if not os.path.isabs(filename): - filename = wikipedia.datafilepath(filename) + filename = wikipedia.config.datafilepath(filename) wikipedia.output(u'Dumping to %s, please wait...' - % wikipedia.shortpath(filename)) + % wikipedia.config.shortpath(filename)) f = bz2.BZ2File(filename, 'w') databases = { 'catContentDB': self.catContentDB, @@ -686,7 +686,7 @@ self.catTitle = catTitle self.catDB = catDB if not os.path.isabs(filename): - filename = wikipedia.datafilepath(filename) + filename = wikipedia.config.datafilepath(filename) self.filename = filename # TODO: make maxDepth changeable with a parameter or config file entry self.maxDepth = maxDepth Modified: trunk/pywikipedia/config.py =================================================================== --- trunk/pywikipedia/config.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/config.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -408,8 +408,6 @@ else: print "WARNING: Skipped '%s': owned by someone else."%_filename -del os, re - # Test for obsoleted and/or unknown variables. for _key in globals().keys(): if _key[0]=='_': @@ -444,6 +442,45 @@ # Save base_dir for use by other modules base_dir = _base_dir + +def makepath(path): + """Return a normalized absolute version of the path argument. + + - if the given path already exists in the filesystem + the filesystem is not modified. + + - otherwise makepath creates directories along the given path + using the dirname() of the path. You may append + a '/' to the path if you want it to be a directory path. + + from holger(a)trillke.net 2002/03/18 + + """ + from os import makedirs + from os.path import normpath, dirname, exists, abspath + + dpath = normpath(dirname(path)) + if not exists(dpath): makedirs(dpath) + return normpath(abspath(path)) + +def datafilepath(*filename): + """Return an absolute path to a data file in a standard location. + + Argument(s) are zero or more directory names, optionally followed by a + data file name. The return path is offset to config.base_dir. Any + directories in the path that do not already exist are created. + + """ + import os + return makepath(os.path.join(base_dir, *filename)) + +def shortpath(path): + """Return a file path relative to config.base_dir.""" + import os + if path.startswith(base_dir): + return path[len(base_dir) + len(os.path.sep) : ] + return path + # # When called as main program, list all configuration variables # @@ -468,3 +505,5 @@ del __sys.modules[__name__].__dict__[__var] del __var, __sys +del os, re + Modified: trunk/pywikipedia/copyright.py =================================================================== --- trunk/pywikipedia/copyright.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/copyright.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -112,7 +112,7 @@ error_color = 'lightred' appdir = "copyright" -output_file = wikipedia.datafilepath(appdir, "output.txt") +output_file = wikipedia.config.datafilepath(appdir, "output.txt") pages_for_exclusion_database = [ ('it', 'Wikipedia:Sospette violazioni di copyright/Lista di esclusione', 'exclusion_list.txt'), @@ -297,7 +297,7 @@ def exclusion_file_list(): for i in pages_for_exclusion_database: - path = wikipedia.datafilepath(appdir, i[0], i[2]) + path = wikipedia.config.datafilepath(appdir, i[0], i[2]) wikipedia.makepath(path) p = wikipedia.Page(wikipedia.getSite(i[0]), i[1]) yield p, path @@ -306,12 +306,14 @@ for page, path in exclusion_file_list(): try: if not os.path.exists(path): - print 'Creating file \'%s\' (%s)' % (wikipedia.shortpath(path), page.aslink()) + print 'Creating file \'%s\' (%s)' % ( + wikipedia.config.shortpath(path), page.aslink()) force_update = True else: file_age = time.time() - os.path.getmtime(path) if file_age > 24 * 60 * 60: - print 'Updating file \'%s\' (%s)' % (wikipedia.shortpath(path), page.aslink()) + print 'Updating file \'%s\' (%s)' % ( + wikipedia.config.shortpath(path), page.aslink()) force_update = True except OSError: raise @@ -384,7 +386,7 @@ result_list.append(entry) result_list += read_file( - wikipedia.datafilepath(appdir, 'exclusion_list.txt'), + wikipedia.config.datafilepath(appdir, 'exclusion_list.txt'), cut_comment = True, cut_newlines = True ).splitlines() @@ -557,7 +559,7 @@ print "** " + entry def exclusion_list_dump(): - f = open(wikipedia.datafilepath(appdir, 'exclusion_list.dump'), 'w') + f = open(wikipedia.config.datafilepath(appdir, 'exclusion_list.dump'), 'w') f.write('\n'.join(excl_list)) f.close() print "Exclusion list dump saved." @@ -984,7 +986,7 @@ % (title.replace(" ", "_").replace("\"", "%22"), id, "author") + output, - wikipedia.datafilepath(appdir, "ID_output.txt")) + wikipedia.config.datafilepath(appdir, "ID_output.txt")) class CheckRobot: def __init__(self, generator): Modified: trunk/pywikipedia/imagecopy.py =================================================================== --- trunk/pywikipedia/imagecopy.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/imagecopy.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -393,7 +393,7 @@ imageP.put(imtxt+u'\n\n{{NowCommons}}', u'{{NowCommons}}') #-etiqueta ok skip view #texto -archivo=wikipedia.datafilepath("Uploadbot.localskips.txt") +archivo=wikipedia.config.datafilepath("Uploadbot.localskips.txt") try: open(archivo, 'r') except IOError: Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/interwiki.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -598,7 +598,7 @@ wikipedia.output(u"NOTE: %s does not have any interwiki links" % self.originPage.aslink(True)) if config.without_interwiki: f = codecs.open( - wikipedia.datafilepath('without_interwiki.txt'), 'a', 'utf-8') + wikipedia.config.datafilepath('without_interwiki.txt'), 'a', 'utf-8') f.write("# %s \n" % page.aslink()) f.close() @@ -745,7 +745,7 @@ if globalvar.autonomous: try: f = codecs.open( - wikipedia.datafilepath('autonomous_problem.dat'), + wikipedia.config.datafilepath('autonomous_problem.dat'), 'a', 'utf-8') f.write("* %s {%s}" % (self.originPage.aslink(True), txt)) if config.interwiki_graph and config.interwiki_graph_url: @@ -1189,7 +1189,7 @@ def dump(self): site = wikipedia.getSite() - dumpfn = wikipedia.datafilepath( + dumpfn = wikipedia.config.datafilepath( 'interwiki-dumps', 'interwikidump-%s-%s.txt' % (site.family.name, site.lang)) f = codecs.open(dumpfn, 'w', 'utf-8') @@ -1547,7 +1547,7 @@ if optRestore or optContinue: site = wikipedia.getSite() - dumpFileName = wikipedia.datafilepath( + dumpFileName = wikipedia.config.datafilepath( 'interwiki-dumps', u'interwikidump-%s-%s.txt' % (site.family.name, site.lang)) Modified: trunk/pywikipedia/login.py =================================================================== --- trunk/pywikipedia/login.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/login.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -172,7 +172,7 @@ The argument data is the raw data, as returned by getCookie(). Returns nothing.""" - filename = wikipedia.datafilepath('login-data', + filename = wikipedia.config.datafilepath('login-data', '%s-%s-%s-login.data' % (self.site.family.name, self.site.lang, self.username)) f = open(filename, 'w') Modified: trunk/pywikipedia/makecat.py =================================================================== --- trunk/pywikipedia/makecat.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/makecat.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -220,7 +220,7 @@ mysite = wikipedia.getSite() wikipedia.setAction(wikipedia.translate(mysite,msg) + ' ' + workingcatname) workingcat = catlib.Category(mysite,mysite.category_namespace()+':'+workingcatname) - filename = wikipedia.datafilepath('category', + filename = wikipedia.config.datafilepath('category', wikipedia.UnicodeToAsciiHtml(workingcatname) + '_exclude.txt') try: f = codecs.open(filename, 'r', encoding = mysite.encoding()) Modified: trunk/pywikipedia/solve_disambiguation.py =================================================================== --- trunk/pywikipedia/solve_disambiguation.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/solve_disambiguation.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -379,7 +379,7 @@ self.enabled = enabled self.ignorelist = [] - filename = wikipedia.datafilepath('disambiguations', + filename = wikipedia.config.datafilepath('disambiguations', self.disambPage.urlname() + '.txt') try: # The file is stored in the disambiguation/ subdir. Create if necessary. @@ -401,7 +401,7 @@ def ignore(self, refPage): if self.enabled: # Skip this occurence next time. - filename = wikipedia.datafilepath('disambiguations', + filename = wikipedia.config.datafilepath('disambiguations', self.disambPage.urlname() + '.txt') try: # Open file for appending. If none exists yet, create a new one. Modified: trunk/pywikipedia/spellcheck.py =================================================================== --- trunk/pywikipedia/spellcheck.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/spellcheck.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -439,7 +439,7 @@ if not checklang: checklang = mysite.language() wikipedia.setAction(wikipedia.translate(mysite,msg)) - filename = wikipedia.datafilepath('spelling', + filename = wikipedia.config.datafilepath('spelling', 'spelling-' + checklang + '.txt') print "Getting wordlist" try: @@ -523,7 +523,7 @@ title = wikipedia.input(u"Which page to check now? (enter to stop)") finally: wikipedia.stopme() - filename = wikipedia.datafilepath('spelling', + filename = wikipedia.config.datafilepath('spelling', 'spelling-' + checklang + '.txt') if rebuild: list = knownwords.keys() Modified: trunk/pywikipedia/splitwarning.py =================================================================== --- trunk/pywikipedia/splitwarning.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/splitwarning.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -1,4 +1,4 @@ -# -*- coding: utf-8 -*- +# -*- coding: utf-8 -*- """Splits a interwiki.log file into chunks of warnings separated by language""" # # (C) Rob W.W. Hooft, 2003 @@ -17,7 +17,7 @@ count={} # TODO: Variable log filename -fn = wikipedia.datafilepath("logs", "interwiki.log") +fn = wikipedia.config.datafilepath("logs", "interwiki.log") logFile = codecs.open(fn, 'r', 'utf-8') rWarning = re.compile('WARNING: (?P<family>.+?): \[\[(?P<code>.+?):.*') for line in logFile: @@ -28,7 +28,7 @@ if code in wikipedia.getSite().languages(): if not files.has_key(code): files[code] = codecs.open( - wikipedia.datafilepath('logs', + wikipedia.config.datafilepath('logs', 'warning-%s-%s.log' % (family, code)), 'w', 'utf-8') count[code] = 0 Modified: trunk/pywikipedia/userinterfaces/terminal_interface.py =================================================================== --- trunk/pywikipedia/userinterfaces/terminal_interface.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/userinterfaces/terminal_interface.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -55,7 +55,7 @@ std_out_handle = ctypes.windll.kernel32.GetStdHandle(-11) csbi = CONSOLE_SCREEN_BUFFER_INFO() ctypes.windll.kernel32.GetConsoleScreenBufferInfo(std_out_handle, ctypes.byref(csbi)) - return csbi.wAttributes & 0x0007 + return (csbi.wAttributes & 0x0007) # TODO: other colors: #0 = Black @@ -86,7 +86,7 @@ } windowsColors = { - 'default': getDefaultTextColorInWindows(), + 'default': 7, 'lightblue': 9, 'lightgreen': 10, 'lightaqua': 11, @@ -95,7 +95,6 @@ 'lightyellow': 14, } - colorTagR = re.compile('\03{(?P<name>%s)}' % '|'.join(windowsColors.keys())) class UI: Modified: trunk/pywikipedia/warnfile.py =================================================================== --- trunk/pywikipedia/warnfile.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/warnfile.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -118,11 +118,11 @@ if os.path.isabs(arg): filename = arg else: - filename = wikipedia.datafilepath("logs", arg) + filename = wikipedia.config.datafilepath("logs", arg) if not filename: mysite = wikipedia.getSite() - filename = wikipedia.datafilepath('logs', + filename = wikipedia.config.datafilepath('logs', 'warning-%s-%s.log' % (mysite.family.name, mysite.lang)) reader = WarnfileReader(filename) bot = WarnfileRobot(reader) Modified: trunk/pywikipedia/watchlist.py =================================================================== --- trunk/pywikipedia/watchlist.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/watchlist.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -34,7 +34,7 @@ # Use cached copy if it exists. watchlist = cache[site] else: - fn = wikipedia.datafilepath('watchlists', + fn = wikipedia.config.datafilepath('watchlists', 'watchlist-%s-%s.dat' % (site.family.name, site.lang)) try: # find out how old our saved dump is (in seconds) @@ -72,14 +72,14 @@ watchlist.append(pageName) # Save the watchlist to disk # The file is stored in the watchlists subdir. Create if necessary. - f = open(wikipedia.datafilepath('watchlists', + f = open(wikipedia.config.datafilepath('watchlists', 'watchlist-%s-%s.dat' % (site.family.name, site.lang)), 'w') pickle.dump(watchlist, f) f.close() def refresh_all(): import dircache, time - filenames = dircache.listdir(wikipedia.datafilepath('watchlists')) + filenames = dircache.listdir(wikipedia.config.datafilepath('watchlists')) watchlist_filenameR = re.compile('watchlist-([a-z\-:]+).dat') for filename in filenames: match = watchlist_filenameR.match(filename) Modified: trunk/pywikipedia/weblinkchecker.py =================================================================== --- trunk/pywikipedia/weblinkchecker.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/weblinkchecker.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -472,7 +472,7 @@ self.reportThread = reportThread site = wikipedia.getSite() self.semaphore = threading.Semaphore() - self.datfilename = wikipedia.datafilepath('deadlinks', + self.datfilename = wikipedia.config.datafilepath('deadlinks', 'deadlinks-%s-%s.dat' % (site.family.name, site.lang)) # Count the number of logged links, so that we can insert captions @@ -500,7 +500,7 @@ isoDate = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime(date)) errorReport += "** In [[%s]] on %s, %s\n" % (pageTitle, isoDate, error) wikipedia.output(u"** Logging link for deletion.") - txtfilename = wikipedia.datafilepath('deadlinks', + txtfilename = wikipedia.config.datafilepath('deadlinks', 'results-%s-%s.txt' % (site.family.name, site.lang)) txtfile = codecs.open(txtfilename, 'a', 'utf-8') self.logCount += 1 Modified: trunk/pywikipedia/welcome.py =================================================================== --- trunk/pywikipedia/welcome.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/welcome.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -449,10 +449,10 @@ wikipedia.output(u'Error! - No fileName!') raise FilenameNotSet("No signature filename specified.") try: - f = codecs.open(wikipedia.datafilepath(fileSignName), 'r', + f = codecs.open(wikipedia.config.datafilepath(fileSignName), 'r', encoding=config.console_encoding) except: - f = codecs.open(wikipedia.datafilepath(fileSignName), 'r', + f = codecs.open(wikipedia.config.datafilepath(fileSignName), 'r', encoding='utf-8') signText = f.read() f.close() @@ -629,7 +629,8 @@ welcomer = u'{{subst:Benvenuto}} %s' welcomed_users = list() - if savedata == True and os.path.exists(wikipedia.datafilepath(filename)): + if savedata == True and os.path.exists( + wikipedia.config.datafilepath(filename)): f = file(filename) number_user = cPickle.load(f) yield number_user Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/wikipedia.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -55,9 +55,6 @@ showDiff(oldtext, newtext): Prints the differences between oldtext and newtext on the screen - datafilepath: Return an absolute path to a data file in a standard - location. - shortpath: Return a relative form of the data file pathname. Wikitext manipulation functions: each of these takes a unicode string containing wiki text as its first argument, and returns a modified version @@ -561,6 +558,10 @@ previously. """ + # TODO: is the description of nofollow_redirects accurate? I can't + # tell where nofollow_redirects is doing anything different than + # get_redirect! + # NOTE: The following few NoPage exceptions could already be thrown at # the Page() constructor. They are raised here instead for convenience, # because all scripts are prepared for NoPage exceptions raised by @@ -767,6 +768,9 @@ """Return text of an old revision of this page; same options as get().""" # TODO: should probably check for bad pagename, NoPage, and other # exceptions that would prevent retrieving text, as get() does + + # TODO: should this default to change_edit_time = False? If we're not + # getting the current version, why change the timestamps? return self._getEditPage( get_redirect=get_redirect, throttle=throttle, sysop=sysop, oldid=oldid, @@ -1163,8 +1167,6 @@ change_edit_time = True, sysop = True) except NoPage: pass - - # if posting to an Esperanto wiki, we must e.g. write Bordeauxx instead # of Bordeaux if self.site().lang == 'eo': @@ -1451,6 +1453,7 @@ """Return a list of Pages that this Page links to. Excludes interwiki and category links. + """ result = [] try: @@ -2577,7 +2580,7 @@ self.setDelay(mindelay) def logfn(self): - return datafilepath('throttle.log') + return config.datafilepath('throttle.log') def checkMultiplicity(self): self.lock.acquire() @@ -3382,7 +3385,7 @@ fam = config.family try: # search for family module in the 'families' subdirectory - sys.path.append(datafilepath('families')) + sys.path.append(config.datafilepath('families')) exec "import %s_family as myfamily" % fam except ImportError: if fatal: @@ -3669,7 +3672,7 @@ else: tmp = '%s-%s-%s-login.data' % ( self.family.name, self.lang, username) - fn = datafilepath('login-data', tmp) + fn = config.datafilepath('login-data', tmp) if not os.path.exists(fn): self._cookies = None self.loginStatusKnown = True @@ -5003,48 +5006,12 @@ nonGlobalArgs.append(arg) return nonGlobalArgs -def makepath(path): - """Return a normalized absolute version of the path argument. - - - if the given path already exists in the filesystem - the filesystem is not modified. - - - otherwise makepath creates directories along the given path - using the dirname() of the path. You may append - a '/' to the path if you want it to be a directory path. - - from holger(a)trillke.net 2002/03/18 - - """ - from os import makedirs - from os.path import normpath, dirname, exists, abspath - - dpath = normpath(dirname(path)) - if not exists(dpath): makedirs(dpath) - return normpath(abspath(path)) - -def datafilepath(*filename): - """Return an absolute path to a data file in a standard location. - - Argument(s) are zero or more directory names, optionally followed by a - data file name. The return path is offset to config.base_dir. Any - directories in the path that do not already exist are created. - - """ - return makepath(os.path.join(config.base_dir, *filename)) - -def shortpath(path): - """Return a file path relative to config.base_dir.""" - if path.startswith(config.base_dir): - return path[len(config.base_dir) + len(os.path.sep) : ] - return path - ######################### # Interpret configuration ######################### # search for user interface module in the 'userinterfaces' subdirectory -sys.path.append(datafilepath('userinterfaces')) +sys.path.append(config.datafilepath('userinterfaces')) exec "import %s_interface as uiModule" % config.userinterface ui = uiModule.UI() verbose = 0 @@ -5263,7 +5230,7 @@ if enabled: if not logname: logname = '%s.log' % calledModuleName() - logfn = datafilepath('logs', logname) + logfn = config.datafilepath('logs', logname) try: logfile = codecs.open(logfn, 'a', 'utf-8') except IOError: @@ -5546,7 +5513,7 @@ # Special opener in case we are using a site with authentication if config.authenticate: import urllib2, cookielib - COOKIEFILE = datafilepath('login-data', 'cookies.lwp') + COOKIEFILE = config.datafilepath('login-data', 'cookies.lwp') cj = cookielib.LWPCookieJar() if os.path.isfile(COOKIEFILE): cj.load(COOKIEFILE) Modified: trunk/pywikipedia/wikipediatools.py =================================================================== --- trunk/pywikipedia/wikipediatools.py 2007-11-06 10:49:56 UTC (rev 4511) +++ trunk/pywikipedia/wikipediatools.py 2007-11-06 21:51:20 UTC (rev 4512) @@ -2,14 +2,18 @@ import os, sys def get_base_dir(): - """ Determine the directory in which user-specific information is stored. - This is determined in the following order - - 1. If the script was called with a -dir: argument, use the directory provided - in this argument - 2. If the user has a PYWIKIBOT_DIR environment variable, use the value of it - 3. If the script was started from a directory that contains a user-config.py - file, use this directory as the base - 4. If all else fails, use the directory from which this module was loaded + """Return the directory in which user-specific information is stored. + + This is determined in the following order - + 1. If the script was called with a -dir: argument, use the directory + provided in this argument + 2. If the user has a PYWIKIBOT_DIR environment variable, use the value + of it + 3. If the script was started from a directory that contains a + user-config.py file, use this directory as the base + 4. If all else fails, use the directory from which this module was + loaded. + """ for arg in sys.argv[1:]: if arg.startswith("-dir:"):

1 0

Add link to new target in solve_disambiguation.py's edit summary
by Erwin 06 Nov '07

06 Nov '07

I'd like to change solve_disambiguation.py to include a link to the new target(s) in the edit summary message when treating disambiguation pages. However this needs a translation of the comments before committing and I'd also like to get some feedback before committing it. I've attached a diff file to a working example of what I want to do. Note this only works for enwiki and nlwiki at the moment. There is still an issue with the number of arguments for the comment format string. Since the comment format string now takes an extra argument custom comments in user-config.py with only one argument won't work anymore. Example edits: http://nl.wikipedia.org/w/index.php?title=Gebruiker:Erwin/Klad1&diff=prev&o… http://nl.wikipedia.org/w/index.php?title=Gebruiker:Erwin/Klad4&diff=prev&o… Erwin85

1 0

Rewrite thoughts (includes replies to the 4507-thread)
by Bryan Tong Minh 06 Nov '07

06 Nov '07

Fucking Gmail. ---------- Forwarded message ---------- From: Bryan Tong Minh <bryan.tongminh(a)gmail.com> Date: Nov 6, 2007 6:17 PM Subject: Re: [Pywikipedia-l] Rewrite thoughts (includes replies to the 4507-thread) To: Merlijn van Deen <valhallasw(a)arctus.nl> On Nov 6, 2007 4:41 PM, Merlijn van Deen <valhallasw(a)arctus.nl> wrote: > > Also the use of persistent HTTP connections makes the framework less > > fool proof. Persistent HTTP connections makes an object that uses them > > automatically unsuitable for sharing between threads. > Is that so? A quick sketchup: > > page.get(): > lock = threading.Lock() #thread safe lock > lock.acquire() > page_get_queue.append((self, query, lock)) > lock.acquire(1) > > with the page getter releasing the lock. But I am probably ignoring some > important thread safety now :) > > The whole point is that the common file object that is associated with the connection is shared between threads. For example, this will happen: def thread1(self): self.sock.sendall('GET / HTTP/1.0\r\n') self.sock.sendall('\r\n') return self.sock.recv(1024) thread.start_new_thread(thread1) thread1() Hell will break lose. Bryan

1 0

← Newer
1
...
22
23
24
25
26
27
28
29
Older →

Jump to page:

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

2008

2007

Pywikipedia-l November 2007