Pywikipedia-svn April 2010

pywikipedia-svn@lists.wikimedia.org

8 participants
96 discussions

SVN: [8110] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 18 Apr '10

18 Apr '10

Revision: 8110 Author: alexsh Date: 2010-04-18 19:42:57 +0000 (Sun, 18 Apr 2010) Log Message: ----------- change all query:logevents to use site().logpages(). logpages():add newer param Modified Paths: -------------- trunk/pywikipedia/imageuncat.py trunk/pywikipedia/redirect.py trunk/pywikipedia/welcome.py trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/imageuncat.py =================================================================== --- trunk/pywikipedia/imageuncat.py 2010-04-18 19:23:24 UTC (rev 8109) +++ trunk/pywikipedia/imageuncat.py 2010-04-18 19:42:57 UTC (rev 8110) @@ -1238,26 +1238,9 @@ today = datetime.utcnow() yesterday = today + timedelta(days=-1) - params = { - 'action' :'query', - 'list' :'logevents', - 'leprop' :'title', - 'letype' :'upload', - 'ledir' :'newer', - 'lelimit' :'5000', - 'lestart' :yesterday.strftime(dateformat), - 'leend' :today.strftime(dateformat) - } - - data = query.GetData(params, site, encodeTitle = False) - try: - for item in data['query']['logevents']: - result.append(item['title']) - except IndexError: - raise NoPage(u'API Error, nothing found in the APIs') - except KeyError: - raise NoPage(u'API Error, nothing found in the APIs') - + for item in site.logpages( number = 5000, mode = 'upload', start = yesterday.strftime(dateformat), + end = today.strftime(dateformat), newer = True, dump = True): + result.append(item['title']) return pagegenerators.PagesFromTitlesGenerator(result, site) def recentChanges(site = None, delay=0, block=70): Modified: trunk/pywikipedia/redirect.py =================================================================== --- trunk/pywikipedia/redirect.py 2010-04-18 19:23:24 UTC (rev 8109) +++ trunk/pywikipedia/redirect.py 2010-04-18 19:42:57 UTC (rev 8110) @@ -463,23 +463,10 @@ start = datetime.datetime.utcnow() \ - datetime.timedelta(0, self.offset*3600) offset_time = start.strftime("%Y%m%d%H%M%S") - params = { - 'action' :'query', - 'list' :'logevents', - 'letype' :'move', - 'leprop' :'title|details', - 'lelimit' : self.api_number, - 'lestart' : offset_time, - } - data = query.GetData(params, encodeTitle = False) - if 'warnings' in data: - raise - allmoves = data['query']['logevents'] wikipedia.output(u'Retrieving %d moved pages via API...' % len(allmoves)) if wikipedia.verbose: wikipedia.output(u"[%s]" % offset_time) - for moved in allmoves: - moved_page = wikipedia.Page(self.site, moved['title']) + for moved_page,u,t,c in self.site.logpages(number = api_number, mode = 'move',start = offset_time): try: if not moved_page.isRedirectPage(): continue Modified: trunk/pywikipedia/welcome.py =================================================================== --- trunk/pywikipedia/welcome.py 2010-04-18 19:23:24 UTC (rev 8109) +++ trunk/pywikipedia/welcome.py 2010-04-18 19:42:57 UTC (rev 8110) @@ -680,26 +680,17 @@ yield x return - params = { - 'action':'query', - 'list':'logevents', - 'letype':'newusers', - 'ledir':'older', - 'leprop':'ids|type|user', - 'lelimit':int(globalvar.queryLimit), - } + starttime = None if globalvar.timeoffset != 0: now = self.site.server_time() - timedelta(minutes=globalvar.timeoffset) - params['lestart'] = int(now.strftime("%Y%m%d%H%M%S")) + starttime = int(now.strftime("%Y%m%d%H%M%S")) elif globalvar.offset != 0: - params['lestart'] = globalvar.offset + starttime = globalvar.offset count_auto = 0 wikipedia.output("Querying new user log from API....") while True: - lev = query.GetData(params, self.site) - count = len(lev['query']['logevents']) - for x in lev['query']['logevents']: + for x in self.site.logpages(number = int(globalvar.queryLimit), mode = 'newusers',start = starttime, dump = True) someone_found = True if 'user' not in x: continue @@ -715,10 +706,6 @@ yield userlib.User(self.site, x['user']) - if count < globalvar.queryLimit and 'query-continue' in lev: - params['lestart'] = lev['query-continue']['logevents']['lestart'] - else: - break if someone_found: if globalvar.quick and count_auto > 0: showStatus() Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2010-04-18 19:23:24 UTC (rev 8109) +++ trunk/pywikipedia/wikipedia.py 2010-04-18 19:42:57 UTC (rev 8110) @@ -5686,7 +5686,7 @@ # TODO: avoid code duplication for the following methods def logpages(self, number = 50, mode = '', title = None, user = None, repeat = False, - namespace = [], start = None, end = None, tag = None, dump = False): + namespace = [], start = None, end = None, tag = None, newer = False, dump = False): if not self.has_api() or self.versionnumber() < 11 or \ mode not in ('block', 'protect', 'rights', 'delete', 'upload', @@ -5707,6 +5707,8 @@ params['lelimit'] = config.special_page_limit if number > 5000 and self.isAllowed('apihighlimits'): params['lelimit'] = 5000 + if newer: + params['ledir'] = 'newer' if user: params['leuser'] = user if title:

1 0

SVN: [8109] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 18 Apr '10

18 Apr '10

Revision: 8109 Author: alexsh Date: 2010-04-18 19:23:24 +0000 (Sun, 18 Apr 2010) Log Message: ----------- change two functions to use Site().logpages() Modified Paths: -------------- trunk/pywikipedia/userlib.py trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/userlib.py =================================================================== --- trunk/pywikipedia/userlib.py 2010-04-18 18:36:22 UTC (rev 8108) +++ trunk/pywikipedia/userlib.py 2010-04-18 19:23:24 UTC (rev 8109) @@ -285,27 +285,10 @@ yield p,t,c,a return - params = { - 'action': 'query', - 'list': 'logevents', - 'letype': 'upload', - 'leuser': self.name(), - 'lelimit': int(number), - } - count = 0 - while True: - data = query.GetData(params, self.site()) - for info in data['query']['logevents']: - count += 1 - yield wikipedia.ImagePage(self.site(), info['title']), info['timestamp'], info['comment'], False + for s in self.site().logpages(number, mode = 'upload', user = self.name(), dump = True) + yield wikipedia.ImagePage(self.site(), s['title']), s['timestamp'], s['comment'], s['pageid'] > 0 + return - if count >= number: - break - if 'query-continue' in data and count < number: - params['lestart'] = data['query-continue']['logevents']['lestart'] - else: - break - def _uploadedImagesOld(self, number = 10): """Yield ImagePages from Special:Log&type=upload""" Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2010-04-18 18:36:22 UTC (rev 8108) +++ trunk/pywikipedia/wikipedia.py 2010-04-18 19:23:24 UTC (rev 8109) @@ -6000,35 +6000,12 @@ No more than 500 (5000 for bots) allowed. Default: 10 """ - params = { - 'action' :'query', - 'list' :'logevents', - 'letype' :'upload', - 'lelimit' :int(number), - } - if lestart: params['lestart'] = lestart - if leend: params['leend'] = leend - if leuser: params['leuser'] = leuser - if letitle: params['letitle'] = letitle - while True: - if self.versionnumber() >= 11: - imagesData = query.GetData(params, self, encodeTitle = False) - else: - raise NotImplementedError("The site version is not support this action.") - - if 'error' in imagesData: - raise RuntimeError('%s' % data['error']) + + for o, u, t, c in self.logpages(number = number, mode = 'upload', title = letitle, user = leuser, + repeat = repeat, start = lestart, end = leend): + yield o, t, u, c + return - for i in imagesData['query']['logevents']: - comment = '' - if 'comment' in i: - comment = i['comment'] - #pageid = i['pageid'] - ##logid = i['logid'] #no use current now - yield ImagePage(self, i['title']), i['timestamp'], i['user'], comment - if not repeat: - break - def recentchanges(self, number = 100, rcstart = None, rcend = None, rcshow = None, rcdir='older', rctype ='edit|new', namespace=None, includeredirects=True, repeat = False): """ Yield ImagePages from APIs, call: action=query&list=recentchanges&rctype=edit|new&rclimit=500

1 0

SVN: [8108] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 18 Apr '10

18 Apr '10

Revision: 8108 Author: alexsh Date: 2010-04-18 18:36:22 +0000 (Sun, 18 Apr 2010) Log Message: ----------- Site().logpages(): add dump mode to return data only. Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2010-04-18 18:26:23 UTC (rev 8107) +++ trunk/pywikipedia/wikipedia.py 2010-04-18 18:36:22 UTC (rev 8108) @@ -5686,7 +5686,7 @@ # TODO: avoid code duplication for the following methods def logpages(self, number = 50, mode = '', title = None, user = None, repeat = False, - namespace = [], start = None, end = None, tag = None): + namespace = [], start = None, end = None, tag = None, dump = False): if not self.has_api() or self.versionnumber() < 11 or \ mode not in ('block', 'protect', 'rights', 'delete', 'upload', @@ -5727,6 +5727,11 @@ for c in result['query']['logevents']: if (not namespace or c['ns'] in namespace) and \ not c.has_key('actionhidden'): + if dump: + # dump result only. + yield c + continue + if c['ns'] == 6: p_ret = ImagePage(self, c['title']) else:

1 0

SVN: [8107] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 18 Apr '10

18 Apr '10

Revision: 8107 Author: alexsh Date: 2010-04-18 18:26:23 +0000 (Sun, 18 Apr 2010) Log Message: ----------- Site().logpages(): typo leprop, add time start-end and tag params Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2010-04-18 17:56:34 UTC (rev 8106) +++ trunk/pywikipedia/wikipedia.py 2010-04-18 18:26:23 UTC (rev 8107) @@ -5685,7 +5685,9 @@ # TODO: avoid code duplication for the following methods - def logpages(self, number = 50, mode = '', title = None, user = None, repeat = False, namespace = [], offset=-1): + def logpages(self, number = 50, mode = '', title = None, user = None, repeat = False, + namespace = [], start = None, end = None, tag = None): + if not self.has_api() or self.versionnumber() < 11 or \ mode not in ('block', 'protect', 'rights', 'delete', 'upload', 'move', 'import', 'patrol', 'merge', 'suppress', @@ -5698,7 +5700,7 @@ 'letype' : mode, 'lelimit' : int(number), 'ledir' : 'older', - 'leprop' : ['ids', 'title', 'type', 'user', 'timestamp', 'comment', 'derails',], + 'leprop' : ['ids', 'title', 'type', 'user', 'timestamp', 'comment', 'details',], } if number > config.special_page_limit: @@ -5709,6 +5711,13 @@ params['leuser'] = user if title: params['letitle'] = title + if start: + params['lestart'] = start + if end: + params['leend'] = end + if tag and self.versionnumber() >= 16: # tag support from mw:r58399 + params['letag'] = tag + nbresults = 0 while True: result = query.GetData(params, self) @@ -5986,8 +5995,6 @@ No more than 500 (5000 for bots) allowed. Default: 10 """ - # def logpages(self, number=50, mode='', user=None, repeat=False, namespace=[], offset=-1): - #return self.logpages(number, mode='upload', user = leuser, repeat) params = { 'action' :'query', 'list' :'logevents',

1 0

SVN: [8106] trunk/pywikipedia/wikipedia.py
by alexsh＠svn.wikimedia.org 18 Apr '10

18 Apr '10

Revision: 8106 Author: alexsh Date: 2010-04-18 17:56:34 +0000 (Sun, 18 Apr 2010) Log Message: ----------- site().logpages(): add title param., version detection, put ImagePage in ns=6 and other default properties. site().newpages(): remove variable api. Modified Paths: -------------- trunk/pywikipedia/wikipedia.py Modified: trunk/pywikipedia/wikipedia.py =================================================================== --- trunk/pywikipedia/wikipedia.py 2010-04-18 12:16:23 UTC (rev 8105) +++ trunk/pywikipedia/wikipedia.py 2010-04-18 17:56:34 UTC (rev 8106) @@ -5685,8 +5685,8 @@ # TODO: avoid code duplication for the following methods - def logpages(self, number=50, mode='', user=None, repeat=False, namespace=[], offset=-1): - if not self.has_api() or \ + def logpages(self, number = 50, mode = '', title = None, user = None, repeat = False, namespace = [], offset=-1): + if not self.has_api() or self.versionnumber() < 11 or \ mode not in ('block', 'protect', 'rights', 'delete', 'upload', 'move', 'import', 'patrol', 'merge', 'suppress', 'review', 'stable', 'gblblock', 'renameuser', @@ -5697,6 +5697,8 @@ 'list' : 'logevents', 'letype' : mode, 'lelimit' : int(number), + 'ledir' : 'older', + 'leprop' : ['ids', 'title', 'type', 'user', 'timestamp', 'comment', 'derails',], } if number > config.special_page_limit: @@ -5705,6 +5707,8 @@ params['lelimit'] = 5000 if user: params['leuser'] = user + if title: + params['letitle'] = title nbresults = 0 while True: result = query.GetData(params, self) @@ -5714,11 +5718,14 @@ for c in result['query']['logevents']: if (not namespace or c['ns'] in namespace) and \ not c.has_key('actionhidden'): - yield (Page(self, c['title'], defaultNamespace=c['ns']), - c['user'], - parsetime2stamp(c['timestamp']), - c['comment'], - ) + if c['ns'] == 6: + p_ret = ImagePage(self, c['title']) + else: + p_ret = Page(self, c['title'], defaultNamespace=c['ns']) + + yield (p_ret, c['user'], + parsetime2stamp(c['timestamp']), + c['comment'], ) nbresults += 1 if nbresults >= number: @@ -5755,9 +5762,8 @@ # should use both offset and limit parameters, and have an # option to fetch older rather than newer pages seen = set() - api = self.has_api() while True: - if api and self.versionnumber() >= 10: + if self.has_api() and self.versionnumber() >= 10: params = { 'action': 'query', 'list': 'recentchanges', @@ -5980,6 +5986,8 @@ No more than 500 (5000 for bots) allowed. Default: 10 """ + # def logpages(self, number=50, mode='', user=None, repeat=False, namespace=[], offset=-1): + #return self.logpages(number, mode='upload', user = leuser, repeat) params = { 'action' :'query', 'list' :'logevents',

1 0

SVN: [8105] trunk/pywikipedia
by alexsh＠svn.wikimedia.org 18 Apr '10

18 Apr '10

Revision: 8105 Author: alexsh Date: 2010-04-18 12:16:23 +0000 (Sun, 18 Apr 2010) Log Message: ----------- cleanup old API detect to Site().has_api() Modified Paths: -------------- trunk/pywikipedia/catlib.py trunk/pywikipedia/upload.py trunk/pywikipedia/watchlist.py Modified: trunk/pywikipedia/catlib.py =================================================================== --- trunk/pywikipedia/catlib.py 2010-04-16 19:12:44 UTC (rev 8104) +++ trunk/pywikipedia/catlib.py 2010-04-18 12:16:23 UTC (rev 8105) @@ -179,13 +179,7 @@ This should not be used outside of this module. """ - try: - if wikipedia.config.use_api and self.site().versionnumber() >= 11: - api_url = self.site().api_address() - del api_url - else: - raise NotImplementedError # version not support - except NotImplementedError: + if not self.site().has_api() or self.site().versionnumber() < 11: for tag, page in self._oldParseCategory(purge, startFrom): yield tag, page return Modified: trunk/pywikipedia/upload.py =================================================================== --- trunk/pywikipedia/upload.py 2010-04-16 19:12:44 UTC (rev 8104) +++ trunk/pywikipedia/upload.py 2010-04-18 12:16:23 UTC (rev 8105) @@ -220,13 +220,7 @@ If the upload fails, the user is asked whether to try again or not. If the user chooses not to retry, returns null. """ - try: - if config.use_api and self.targetSite.versionnumber() >= 16: - x = self.targetSite.api_address() - del x - else: - raise NotImplementedError - except NotImplementedError: + if not self.targetSite.has_api() or self.targetSite.versionnumber() < 16: return self._uploadImageOld(debug) if not hasattr(self,'_contents'): Modified: trunk/pywikipedia/watchlist.py =================================================================== --- trunk/pywikipedia/watchlist.py 2010-04-16 19:12:44 UTC (rev 8104) +++ trunk/pywikipedia/watchlist.py 2010-04-18 12:16:23 UTC (rev 8105) @@ -59,13 +59,7 @@ return pageName in watchlist def refresh(site, sysop=False): - try: - if wikipedia.config.use_api and site.versionnumber() >= 10: - x = site.api_address() - del x - else: - raise NotImplementedError - except NotImplementedError: + if not site.has_api() or site.versionnumber() < 10: _refreshOld(site) # get watchlist special page's URL @@ -79,7 +73,7 @@ 'wlprop': 'title', } - wikipedia.output(u'Retrieving watchlist for %s' % repr(site)) + wikipedia.output(u'Retrieving watchlist for %s via API.' % repr(site)) #wikipedia.put_throttle() # It actually is a get, but a heavy one. watchlist = [] while True:

1 0

SVN: [8104] branches/rewrite/generate_user_files.py
by russblau＠svn.wikimedia.org 16 Apr '10

16 Apr '10

Revision: 8104 Author: russblau Date: 2010-04-16 19:12:44 +0000 (Fri, 16 Apr 2010) Log Message: ----------- Updated for use with rewrite branch. Modified Paths: -------------- branches/rewrite/generate_user_files.py Modified: branches/rewrite/generate_user_files.py =================================================================== --- branches/rewrite/generate_user_files.py 2010-04-16 13:43:18 UTC (rev 8103) +++ branches/rewrite/generate_user_files.py 2010-04-16 19:12:44 UTC (rev 8104) @@ -2,9 +2,51 @@ """ Script to create user files (user-config.py, user-fixes.py) """ __version__ = '$Id$' -import os, sys, codecs, re +import codecs, os, platform, re, sys -base_dir = '' +def get_base_dir(): + """Return the directory in which user-specific information is stored. + + This is determined in the following order - + 1. If the script was called with a -dir: argument, use the directory + provided in this argument + 2. If the user has a PYWIKIBOT2_DIR environment variable, use the value + of it + 3. Use (and if necessary create) a 'pywikibot' folder (Windows) or + '.pywikibot' directory (Unix and similar) under the user's home + directory. + + """ + # copied from config2.py, without the lines that check whether the + # directory already contains a user-config.py file + # this code duplication is nasty, should fix + NAME = "pywikibot" + for arg in sys.argv[1:]: + if arg.startswith("-dir:"): + base_dir = arg[5:] + sys.argv.remove(arg) + break + else: + if "PYWIKIBOT2_DIR" in os.environ: + base_dir = os.environ["PYWIKIBOT2_DIR"] + else: + is_windows = sys.platform == 'win32' + home = os.path.expanduser("~") + if is_windows: + _win_version = int(platform.version()[0]) + if _win_version == 5: + base_dir = os.path.join(home, "Application Data", NAME) + elif _win_version == 6: + base_dir = os.path.join(home, "AppData\\Roaming", NAME) + else: + base_dir = os.path.join(home, "."+NAME) + if not os.path.isdir(base_dir): + os.makedirs(base_dir, mode=0700) + if not os.path.isabs(base_dir): + base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir)) + return base_dir + +base_dir = get_base_dir() console_encoding = sys.stdout.encoding if console_encoding is None or sys.platform == 'cygwin': @@ -35,6 +77,47 @@ print("Invalid response") return response +def change_base_dir(): + """Create a new user directory.""" + global base_dir + while True: + new_base = raw_input("New user directory? ") + new_base = os.path.abspath(new_base) + if os.path.exists(new_base): + if os.path.isfile(new_base): + print("ERROR: there is an existing file with that name.") + continue + # make sure user can read and write this directory + if not os.access(new_base, os.R_OK|os.W_OK): + print("ERROR: directory access restricted") + continue + print("OK: using existing directory") + break + else: + try: + os.mkdir(new_base, 0700) + except Exception: + print("ERROR: directory creation failed") + continue + print("OK: Created new directory.") + break + + from textwrap import wrap + msg = wrap("""WARNING: Your user files will be created in the directory +'%(new_base)s' you have chosen. To access these files, you will either have +to use the argument "-dir:%(new_base)s" every time you run the bot, or set +the environment variable "PYWIKIBOT2_DIR" equal to this directory name in +your operating system. See your operating system documentation for how to +set environment variables.""" % locals(), width=76) + for line in msg: + print line + ok = raw_input("Is this OK? [y/N] ") + if ok in ["Yy"]: + base_dir = new_base + return True + print "Aborting changes." + return False + def file_exists(filename): if os.path.exists(filename): print("'%s' already exists." % filename) @@ -44,19 +127,32 @@ def create_user_config(): _fnc = os.path.join(base_dir, "user-config.py") if not file_exists(_fnc): - know_families = re.findall(r'(.+)_family.py\b', '\n'.join(os.listdir(os.path.join(base_dir, "families")))) - fam = listchoice(know_families, "Select family of sites we are working on", default = 'wikipedia') - mylang = raw_input("The language code of the site we're working on (default: 'en'): ") or 'en' - username = raw_input("Username (%s %s): " % (mylang, fam)) or 'UnnamedBot' + known_families = re.findall(r'(.+)_family.py\b', + '\n'.join(os.listdir(os.path.join(base_dir, "families")))) + fam = listchoice(known_families, + "Select family of sites we are working on", + default='wikipedia') + mylang = raw_input( +"The language code of the site we're working on (default: 'en'): ") or 'en' + username = raw_input("Username (%s %s): " % (mylang, fam) + ) or 'UnnamedBot' username = unicode(username, console_encoding) while True: - choice = raw_input("Which variant of user_config.py:\n[S]mall or [E]xtended (with further informations)? ").upper() - if choice in ['S','E']: + choice = raw_input( +"Which variant of user_config.py:\n[S]mall or [E]xtended (with further information)? " + ).upper() + if choice in "SE": break # # I don't like this solution. Temporary for me. - f = codecs.open("config.py", "r", "utf-8") ; cpy = f.read() ; f.close() + # + # determine what directory this script (generate_user_files.py) lives in + install = os.path.dirname(os.path.abspath(sys.argv[0])) + # config2.py will be in the pywikibot/ directory + f = codecs.open(os.path.join(install, "pywikibot", "config2.py"), + "r", "utf-8") + cpy = f.read() ; f.close() res = re.findall("^(############## (?:LOGFILE|" "INTERWIKI|" @@ -67,14 +163,16 @@ "DATABASE|" "SEARCH ENGINE|" "COPYRIGHT|" - "FURTHER) SETTINGS .*?)^(?=#####|# =====)", cpy, re.MULTILINE | re.DOTALL) + "FURTHER) SETTINGS .*?)^(?=#####|# =====)", + cpy, re.MULTILINE | re.DOTALL) config_text = '\n'.join(res) f = codecs.open(_fnc, "w", "utf-8") if choice == 'E': f.write("""# -*- coding: utf-8 -*- -# This is an automatically generated file. You can find more configuration parameters in 'config.py' file. +# This is an automatically generated file. You can find more configuration +# parameters in 'config.py' file. # The family of sites we are working on. wikipedia.py will import # families/xxx_family.py so if you want to change this variable, @@ -124,16 +222,51 @@ print("'%s' written." % _fnf) if __name__ == "__main__": - print("1: Create user_config.py file") - print("2: Create user_fixes.py file") - print("3: The two files") - choice = raw_input("What do you do? ") - if choice == "1": - create_user_config() - if choice == "2": - create_user_fixes() - if choice == "3": - create_user_config() - create_user_fixes() - if not choice in ["1", "2", "3"]: - print("Nothing to do") + while True: + print("Your default user directory is '%s'" % base_dir) + ok = raw_input("[K]eep/Change? ").upper().strip() + if (not ok) or "KEEP".startswith(ok): + break + if "CHANGE".startswith(ok): + if change_base_dir(): + break + while True: + if os.path.exists(os.path.join(base_dir, "user-config.py")): + break + do_copy = raw_input( + "Do you want to copy user files from an existing pywikipedia installation? " + ).upper().strip() + if do_copy and "YES".startswith(do_copy): + oldpath = raw_input("Path to existing wikipedia.py? ") + if not os.path.exists(oldpath): + print("ERROR: Not a valid path") + continue + if os.path.isfile(oldpath): + # User probably typed /wikipedia.py at the end, so strip it + oldpath = os.path.dirname(oldpath) + if not os.path.isfile(os.path.join(oldpath, "user-config.py")): + print("ERROR: no user_config.py found in that directory") + continue + newf = file(os.path.join(base_dir, "user-config.py"), "wb") + oldf = file(os.path.join(oldpath, "user-config.py"), "rb") + newf.write(oldf.read()) + newf.close() ; oldf.close() + if os.path.isfile(os.path.join(oldpath, "user-fixes.py")): + newfix = file(os.path.join(base_dir, "user-fixes.py"), "wb") + oldfix = file(os.path.join(oldpath, "user-fixes.py"), "rb") + newfix.write(oldfix.read()) + newfix.close() ; oldfix.close() + elif do_copy and "NO".startswith(do_copy): + break + if not os.path.isfile(os.path.join(base_dir, "user-config.py")): + a = raw_input("Create user-config.py file? [y/N] ") + if a[:1] in ["Y", "y"]: + create_user_config() + else: + print("NOTE: user-config.py already exists in the directory") + if not os.path.isfile(os.path.join(base_dir, "user-fixes.py")): + a = raw_input("Create user-fixes.py file? [y/N] ") + if a[:1] in ["Y", "y"]: + create_user_fixes() + else: + print("NOTE: user-fixes.py already exists in the directory")

1 0

SVN: [8103] branches/rewrite/generate_user_files.py
by russblau＠svn.wikimedia.org 16 Apr '10

16 Apr '10

Revision: 8103 Author: russblau Date: 2010-04-16 13:43:18 +0000 (Fri, 16 Apr 2010) Log Message: ----------- Copy for porting to rewrite branch. Added Paths: ----------- branches/rewrite/generate_user_files.py Copied: branches/rewrite/generate_user_files.py (from rev 8102, trunk/pywikipedia/generate_user_files.py) =================================================================== --- branches/rewrite/generate_user_files.py (rev 0) +++ branches/rewrite/generate_user_files.py 2010-04-16 13:43:18 UTC (rev 8103) @@ -0,0 +1,139 @@ +# -*- coding: utf-8 -*- +""" Script to create user files (user-config.py, user-fixes.py) """ +__version__ = '$Id$' + +import os, sys, codecs, re + +base_dir = '' +console_encoding = sys.stdout.encoding + +if console_encoding is None or sys.platform == 'cygwin': + console_encoding = "iso-8859-1" + +def listchoice(clist = [], message = None, default = None): + + if not message: + message = "Select" + + if default: + message += " (default: %s)" % default + + message += ": " + + for n, i in enumerate(clist): + print ("%d: %s" % (n + 1, i)) + + while True: + choice = raw_input(message) + + if choice == '' and default: + return default + + try: + return clist[int(choice) - 1] + except: + print("Invalid response") + return response + +def file_exists(filename): + if os.path.exists(filename): + print("'%s' already exists." % filename) + return True + return False + +def create_user_config(): + _fnc = os.path.join(base_dir, "user-config.py") + if not file_exists(_fnc): + know_families = re.findall(r'(.+)_family.py\b', '\n'.join(os.listdir(os.path.join(base_dir, "families")))) + fam = listchoice(know_families, "Select family of sites we are working on", default = 'wikipedia') + mylang = raw_input("The language code of the site we're working on (default: 'en'): ") or 'en' + username = raw_input("Username (%s %s): " % (mylang, fam)) or 'UnnamedBot' + username = unicode(username, console_encoding) + while True: + choice = raw_input("Which variant of user_config.py:\n[S]mall or [E]xtended (with further informations)? ").upper() + if choice in ['S','E']: + break + + # + # I don't like this solution. Temporary for me. + f = codecs.open("config.py", "r", "utf-8") ; cpy = f.read() ; f.close() + + res = re.findall("^(############## (?:LOGFILE|" + "INTERWIKI|" + "SOLVE_DISAMBIGUATION|" + "IMAGE RELATED|" + "TABLE CONVERSION BOT|" + "WEBLINK CHECKER|" + "DATABASE|" + "SEARCH ENGINE|" + "COPYRIGHT|" + "FURTHER) SETTINGS .*?)^(?=#####|# =====)", cpy, re.MULTILINE | re.DOTALL) + config_text = '\n'.join(res) + + f = codecs.open(_fnc, "w", "utf-8") + if choice == 'E': + f.write("""# -*- coding: utf-8 -*- + +# This is an automatically generated file. You can find more configuration parameters in 'config.py' file. + +# The family of sites we are working on. wikipedia.py will import +# families/xxx_family.py so if you want to change this variable, +# you need to write such a file. +family = '%s' + +# The language code of the site we're working on. +mylang = '%s' + +# The dictionary usernames should contain a username for each site where you +# have a bot account. +usernames['%s']['%s'] = u'%s' + + +%s""" % (fam, mylang, fam, mylang, username, config_text)) + else: + f.write("""# -*- coding: utf-8 -*- +family = '%s' +mylang = '%s' +usernames['%s']['%s'] = u'%s' +""" % (fam, mylang, fam, mylang, username)) + f.close() + print("'%s' written." % _fnc) + +def create_user_fixes(): + _fnf = os.path.join(base_dir, "user-fixes.py") + if not file_exists(_fnf): + f = codecs.open(_fnf, "w", "utf-8") + f.write(r"""# -*- coding: utf-8 -*- + +# +# This is only an example. Don't use it. +# + +fixes['example'] = { + 'regex': True, + 'msg': { + '_default':u'no summary specified', + }, + 'replacements': [ + (ur'\bword\b', u'two words'), + ] +} + +""") + f.close() + print("'%s' written." % _fnf) + +if __name__ == "__main__": + print("1: Create user_config.py file") + print("2: Create user_fixes.py file") + print("3: The two files") + choice = raw_input("What do you do? ") + if choice == "1": + create_user_config() + if choice == "2": + create_user_fixes() + if choice == "3": + create_user_config() + create_user_fixes() + if not choice in ["1", "2", "3"]: + print("Nothing to do")

1 0

SVN: [8102] trunk/pywikipedia/userinterfaces/transliteration.py
by malafaya＠svn.wikimedia.org 16 Apr '10

16 Apr '10

Revision: 8102 Author: malafaya Date: 2010-04-16 10:28:09 +0000 (Fri, 16 Apr 2010) Log Message: ----------- * Some Korean translit Modified Paths: -------------- trunk/pywikipedia/userinterfaces/transliteration.py Modified: trunk/pywikipedia/userinterfaces/transliteration.py =================================================================== --- trunk/pywikipedia/userinterfaces/transliteration.py 2010-04-16 09:08:26 UTC (rev 8101) +++ trunk/pywikipedia/userinterfaces/transliteration.py 2010-04-16 10:28:09 UTC (rev 8102) @@ -1405,6 +1405,18 @@ self.trans[char] = u"ga" for char in u"리": self.trans[char] = u"ri" + for char in u"그": + self.trans[char] = u"geu" + for char in u"지": + self.trans[char] = u"ji" + for char in u"야": + self.trans[char] = u"ya" + for char in u"바": + self.trans[char] = u"ba" + for char in u"슈": + self.trans[char] = u"syu" + for char in u"키": + self.trans[char] = u"ki" def transliterate(self, char, default="?", prev="-", next="-"):

1 0

SVN: [8101]
by xqt＠svn.wikimedia.org 16 Apr '10

16 Apr '10

Revision: 8101 Author: xqt Date: 2010-04-16 09:08:26 +0000 (Fri, 16 Apr 2010) Log Message: ----------- use unicode strings, minor changes Modified Paths: -------------- branches/rewrite/scripts/interwiki.py trunk/pywikipedia/interwiki.py Modified: branches/rewrite/scripts/interwiki.py =================================================================== --- branches/rewrite/scripts/interwiki.py 2010-04-16 08:47:13 UTC (rev 8100) +++ branches/rewrite/scripts/interwiki.py 2010-04-16 09:08:26 UTC (rev 8101) @@ -1645,7 +1645,7 @@ except pywikibot.NoPage: pywikibot.output(u"BUG>>> %s no longer exists?" % new[site]) continue - mods, adding, removing, modifying = compareLanguages(old, new, insite = lclSite) + mods, mcomment, adding, removing, modifying = compareLanguages(old, new, insite = lclSite) if (len(removing) > 0 and not globalvar.autonomous) or (len(modifying) > 0 and self.problemfound) or len(old) == 0 or (globalvar.needlimit and len(adding) + len(modifying) >= globalvar.needlimit +1): try: if self.replaceLinks(new[site], new, bot): @@ -1781,8 +1781,8 @@ pywikibot.output(u"Changes to be made: %s" % mods) oldtext = page.get() template = (page.namespace() == 10) - newtext = pywikibot.replaceLanguageLinks(oldtext, new, - site = page.site(), + newtext = pywikibot.replaceLanguageLinks(oldtext, new, + site = page.site(), template = template) # This is for now. Later there should be different funktions for each kind if not botMayEdit(page): @@ -1845,7 +1845,7 @@ timeout=60 while 1: try: - status, reason, data = page.put(newtext, comment = pywikibot.translate(page.site().lang, msg)[0] + mods) + status, reason, data = page.put(newtext, comment = mcomment) except pywikibot.LockedPage: pywikibot.output(u'Page %s is locked. Skipping.' % (page,)) raise SaveError(u'Locked') Modified: trunk/pywikipedia/interwiki.py =================================================================== --- trunk/pywikipedia/interwiki.py 2010-04-16 08:47:13 UTC (rev 8100) +++ trunk/pywikipedia/interwiki.py 2010-04-16 09:08:26 UTC (rev 8101) @@ -1156,14 +1156,14 @@ pywikibot.output(u"NOTE: Ignoring non-disambiguation page %s for %s because disambiguation page %s has already been found." % (page.aslink(True), self.originPage.aslink(True), disambig.aslink(True))) return (True, None) else: - choice = pywikibot.inputChoice('WARNING: %s is a disambiguation page, but %s doesn\'t seem to be one. Follow it anyway?' % (self.originPage.aslink(True), page.aslink(True)), ['Yes', 'No', 'Add an alternative', 'Give up'], ['y', 'n', 'a', 'g']) + choice = pywikibot.inputChoice(u'WARNING: %s is a disambiguation page, but %s doesn\'t seem to be one. Follow it anyway?' % (self.originPage.aslink(True), page.aslink(True)), ['Yes', 'No', 'Add an alternative', 'Give up'], ['y', 'n', 'a', 'g']) elif not self.originPage.isDisambig() and page.isDisambig(): nondisambig = self.getFoundNonDisambig(page.site()) if nondisambig: pywikibot.output(u"NOTE: Ignoring disambiguation page %s for %s because non-disambiguation page %s has already been found." % (page.aslink(True), self.originPage.aslink(True), nondisambig.aslink(True))) return (True, None) else: - choice = pywikibot.inputChoice('WARNING: %s doesn\'t seem to be a disambiguation page, but %s is one. Follow it anyway?' % (self.originPage.aslink(True), page.aslink(True)), ['Yes', 'No', 'Add an alternative', 'Give up'], ['y', 'n', 'a', 'g']) + choice = pywikibot.inputChoice(u'WARNING: %s doesn\'t seem to be a disambiguation page, but %s is one. Follow it anyway?' % (self.originPage.aslink(True), page.aslink(True)), ['Yes', 'No', 'Add an alternative', 'Give up'], ['y', 'n', 'a', 'g']) if choice == 'n': return (True, None) elif choice == 'a': @@ -1191,7 +1191,7 @@ if config.without_interwiki: f = codecs.open( pywikibot.config.datafilepath('without_interwiki.txt'), 'a', 'utf-8') - f.write("# %s \n" % page.aslink(forceInterwiki=True)) + f.write(u"# %s \n" % page.aslink(forceInterwiki=True)) f.close() def askForHints(self, counter): @@ -1317,7 +1317,7 @@ pywikibot.output(u"%s: %s gives new %sredirect %s" % (self.originPage.aslink(), page.aslink(True), redir, redirectTargetPage.aslink(True))) continue - + # must be behind the page.isRedirectPage() part # otherwise a redirect error would be raised if page.isEmpty() and not page.isCategory(): @@ -1381,10 +1381,10 @@ f = codecs.open( pywikibot.config.datafilepath('autonomous_problems.dat'), 'a', 'utf-8') - f.write("* %s {Found more than one link for %s}" % (self.originPage.aslink(True), page.site())) + f.write(u"* %s {Found more than one link for %s}" % (self.originPage.aslink(True), page.site())) if config.interwiki_graph and config.interwiki_graph_url: filename = interwiki_graph.getFilename(self.originPage, extension = config.interwiki_graph_formats[0]) - f.write(" [%s%s graph]" % (config.interwiki_graph_url, filename)) + f.write(u" [%s%s graph]" % (config.interwiki_graph_url, filename)) f.write("\n") f.close() # FIXME: What errors are we catching here? @@ -1460,7 +1460,7 @@ site = page.site() if site == self.originPage.site(): if page != self.originPage: - self.problem("Found link to %s" % page.aslink(True) ) + self.problem(u"Found link to %s" % page.aslink(True) ) self.whereReport(page) errorCount += 1 else: @@ -1473,7 +1473,7 @@ for site, pages in new.iteritems(): if len(pages) > 1: errorCount += 1 - self.problem("Found more than one link for %s" % site) + self.problem(u"Found more than one link for %s" % site) if not errorCount and not globalvar.select: # no errors, so all lists have only one item @@ -1755,7 +1755,9 @@ pywikibot.output(u"Changes to be made: %s" % mods) oldtext = page.get() template = (page.namespace() == 10) - newtext = pywikibot.replaceLanguageLinks(oldtext, new, site = page.site(), template = template) + newtext = pywikibot.replaceLanguageLinks(oldtext, new, + site = page.site(), + template = template) # This is for now. Later there should be different funktions for each kind if not botMayEdit(page): if template: @@ -1771,7 +1773,7 @@ # Determine whether we need permission to submit ask = False if removing and removing != [page.site()]: # Allow for special case of a self-pointing interwiki link - self.problem('Found incorrect link to %s in %s'% (",".join([x.lang for x in removing]), page.aslink(True)), createneed = False) + self.problem(u'Found incorrect link to %s in %s'% (",".join([x.lang for x in removing]), page.aslink(True)), createneed = False) ask = True if globalvar.force: ask = False @@ -1791,7 +1793,7 @@ page.site().hostname(), page.site().nice_get_address(page.title()) )) - pywikibot.input("Press Enter when finished in browser.") + pywikibot.input(u"Press Enter when finished in browser.") return True elif answer == 'a': # don't ask for the rest of this subject @@ -1854,7 +1856,7 @@ elif answer == 'g': raise GiveUpOnPage else: - raise LinkMustBeRemoved('Found incorrect link to %s in %s'% (",".join([x.lang for x in removing]), page.aslink(True))) + raise LinkMustBeRemoved(u'Found incorrect link to %s in %s'% (",".join([x.lang for x in removing]), page.aslink(True))) def reportBacklinks(self, new, updatedSites): """

1 0

← Newer
1
2
3
4
5
6
7
8
9
10
Older →

2024

2023

2022

2021

2020

2019

2018

2017

2016

2015

2014

2013

2012

2011

2010

2009

Pywikipedia-svn April 2010