Revision: 5643 Author: russblau Date: 2008-06-26 21:39:01 +0000 (Thu, 26 Jun 2008)

Log Message: ----------- Using keyword substitution in string formatting; will make life easier later when messages are translated, since word order may not be the same in all languages.

Modified Paths: -------------- branches/rewrite/pywikibot/__init__.py branches/rewrite/pywikibot/comms/http.py branches/rewrite/pywikibot/comms/threadedhttp.py branches/rewrite/pywikibot/config.py branches/rewrite/pywikibot/family.py branches/rewrite/pywikibot/login.py branches/rewrite/pywikibot/throttle.py

Modified: branches/rewrite/pywikibot/__init__.py =================================================================== --- branches/rewrite/pywikibot/__init__.py 2008-06-26 21:28:02 UTC (rev 5642) +++ branches/rewrite/pywikibot/__init__.py 2008-06-26 21:39:01 UTC (rev 5643) @@ -48,12 +48,12 @@ try: exec "from site import %s as __Site" % interface except ImportError: - raise ValueError("Invalid interface name '%s'" % interface) + raise ValueError("Invalid interface name '%(interface)s'" % locals()) key = '%s:%s:%s' % (fam, code, user) if not _sites.has_key(key): _sites[key] = __Site(code=code, fam=fam, user=user) - logger.debug("Instantiating Site object '%s'" - % _sites[key]) + logger.debug("Instantiating Site object '%(site)s'" + % {'site': _sites[key]}) return _sites[key]

getSite = Site # alias for backwards-compability @@ -68,7 +68,7 @@

def input(prompt, password=False): if isinstance(prompt, unicode): - prompt = prompt.encode(sys.stdout.encoding, "replace") + prompt = prompt.encode(sys.stdout.encoding, "xmlcharrefreplace") if password: import getpass return getpass.getpass(prompt) @@ -104,6 +104,7 @@ # Throttle and thread handling

threadpool = [] # add page-putting threads to this list as they are created +stopped = False

def stopme(): """Drop this process from the throttle log, after pending threads finish. @@ -112,13 +113,16 @@ at Python exit.

""" + global stopped + if stopped: + return logger = logging.getLogger("wiki")

logger.debug("stopme() called") - threadcount = sum(1 for thd in threadpool if thd.isAlive()) - if threadcount: - logger.info("Waiting for approximately %s threads to finish." - % threadcount) + count = sum(1 for thd in threadpool if thd.isAlive()) + if count: + logger.info("Waiting for about %(count)s pages to be saved." + % locals()) for thd in threadpool: if thd.isAlive(): thd.join() @@ -126,6 +130,7 @@ try: _sites[_sites.keys()[0]].throttle.drop() logger.info("Dropped throttle(s).") + stopped = True except IndexError: pass

Modified: branches/rewrite/pywikibot/comms/http.py =================================================================== --- branches/rewrite/pywikibot/comms/http.py 2008-06-26 21:28:02 UTC (rev 5642) +++ branches/rewrite/pywikibot/comms/http.py 2008-06-26 21:39:01 UTC (rev 5643) @@ -57,7 +57,7 @@

# Build up HttpProcessors -logger.info('Starting %i threads...' % numthreads) +logger.info('Starting %(numthreads)i threads...' % locals()) for i in range(numthreads): proc = threadedhttp.HttpProcessor(http_queue, cookie_jar, connection_pool) proc.setDaemon(True) @@ -94,13 +94,14 @@ http_queue.put(request) request.lock.acquire()

- #do some error correcting stuff + #TODO: do some error correcting stuff

#if all else fails if isinstance(request.data, Exception): raise request.data

if request.data[0].status != 200: - logger.warning("Http response status %s" % request.data[0].status) + logger.warning("Http response status %(status)s" + % {'status': request.data[0].status})

return request.data[1]

Modified: branches/rewrite/pywikibot/comms/threadedhttp.py =================================================================== --- branches/rewrite/pywikibot/comms/threadedhttp.py 2008-06-26 21:28:02 UTC (rev 5642) +++ branches/rewrite/pywikibot/comms/threadedhttp.py 2008-06-26 21:39:01 UTC (rev 5643) @@ -38,7 +38,7 @@ import pkg_resources except ImportError: logger.critical( - "Error : You need the python module setuptools to use this module") + "Error: You need the python module setuptools to use this module") sys.exit(1) pkg_resources.require("httplib2") import httplib2 @@ -67,6 +67,8 @@ for key in self.connections: for connection in self.connections[key]: connection.close() + except AttributeError: + pass # this shows up when logger has been destroyed first finally: self.lock.release()

Modified: branches/rewrite/pywikibot/config.py =================================================================== --- branches/rewrite/pywikibot/config.py 2008-06-26 21:28:02 UTC (rev 5642) +++ branches/rewrite/pywikibot/config.py 2008-06-26 21:39:01 UTC (rev 5643) @@ -8,6 +8,7 @@

import os, re import sys as __sys + # IMPORTANT: # Do not change any of the variables in this file. Instead, make # a file user-config.py, and overwrite values in there. @@ -108,10 +109,11 @@ base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir)) # make sure this path is valid and that it contains user-config file if not os.path.isdir(base_dir): - raise RuntimeError("Directory '%s' does not exist." % base_dir) + raise RuntimeError("Directory '%(base_dir)s' does not exist." + % locals()) if not os.path.exists(os.path.join(base_dir, "user-config.py")): - raise RuntimeError("No user-config.py found in directory '%s'." - % base_dir) + raise RuntimeError("No user-config.py found in directory '%(base_dir)s'." + % locals()) return base_dir

_base_dir = _get_base_dir() @@ -183,8 +185,7 @@ _key2 = _winreg.OpenKey(_winreg.HKEY_CLASSES_ROOT, '%s\shell\open\command' % _progID) _cmd = _winreg.QueryValueEx(_key2, None)[0] editor = _cmd.replace('%1', '') - # Notepad is even worse than our Tkinter editor. Nobody has - # deserved to use it. + # Notepad is even worse than our Tkinter editor. if editor.lower().endswith('notepad.exe'): editor = None except: @@ -319,7 +320,7 @@ # That can do very ugly results. deIndentTables = True # table2wiki.py works quite stable, so you might switch to True -table2wikiAskOnlyWarnings = True +table2wikiAskOnlyWarnngs = True table2wikiSkipWarnings = False

############## WEBLINK CHECKER SETTINGS ############## @@ -462,9 +463,11 @@ if __sys.platform=='win32' or _filemode&002==0: execfile(_filename) else: - print "WARNING: Skipped '%s': writeable by others."%_filename + print "WARNING: Skipped '%(fn)s': writeable by others."\ + % {'fn' :_filename} else: - print "WARNING: Skipped '%s': owned by someone else."%_filename + print "WARNING: Skipped '%(fn)s': owned by someone else."\ + % {'fn' :_filename}

# Test for obsoleted and/or unknown variables. for _key in globals().keys(): @@ -484,12 +487,14 @@ elif ot==type(1) and nt==type(True): pass else: - print "WARNING: Type of '%s' changed"%_key - print " Was: ",ot - print " Now: ",nt + print "WARNING: Type of '%(_key)s' changed" % locals() + print " %(was)s: %(old)s" % {'was': "Was", 'old': ot} + print " %(was)s: %(new)s" % {'now': "Now", 'new': nt} del nt,ot else: - print "WARNING: Configuration variable %r is defined but unknown. Misspelled?" %_key + logger.warn( + "Configuration variable %(_key)r is defined but unknown. Misspelled?" + % locals())

# Fix up default console_encoding if console_encoding == None: @@ -549,7 +554,7 @@ if _arg=="modified": _all=0 else: - print "Unknown arg %s ignored"%_arg + print "Unknown arg %(_arg)s ignored" % locals() _k=globals().keys() _k.sort() for _name in _k:

Modified: branches/rewrite/pywikibot/family.py =================================================================== --- branches/rewrite/pywikibot/family.py 2008-06-26 21:28:02 UTC (rev 5642) +++ branches/rewrite/pywikibot/family.py 2008-06-26 21:39:01 UTC (rev 5643) @@ -2505,11 +2505,17 @@ elif fallback: return self.linktrails[fallback] else: - raise KeyError('ERROR: linktrail in language %s unknown' % code) + raise KeyError( + "ERROR: linktrail in language %(language_code)s unknown" + % {'language_code': code})

- def namespace(self, code, ns_number, fallback = '_default', all = False): + def namespace(self, code, ns_number, fallback='_default', all=False): if not self.isDefinedNS(ns_number): - raise KeyError('ERROR: Unknown namespace %d for %s:%s' % (ns_number, code, self.name)) + raise KeyError( +'ERROR: Unknown namespace %(ns_number)d for %(language_code)s:%(ns_name)s' + % {'ns_number': ns_number, + 'language_code': code, + 'ns_name': self.name}) elif self.isNsI18N(ns_number, code): v = self.namespaces[ns_number][code] if type(v) is not list: @@ -2525,35 +2531,35 @@ if type(v) is not list: v = [v,] else: - raise KeyError('ERROR: title for namespace %d in language %s unknown' % (ns_number, code)) - + raise KeyError( +'ERROR: title for namespace %(ns_number)d in language %(language_code)s unknown' + % {'ns_number': ns_number, + 'language_code': code}) if all: - namespaces = [] - - # Unique list - for ns in v: - if ns not in namespaces: - namespaces.append(ns) - + namespaces = list(set(v)) # Lowercase versions of namespaces if code not in self.nocapitalize: - namespaces.extend([ns[0].lower() + ns[1:] for ns in namespaces if ns and ns[0].lower() != ns[0].upper()]) - + namespaces.extend([ns[0].lower() + ns[1:] + for ns in namespaces + if ns and ns[0].lower() != ns[0].upper()]) # Underscore versions of namespaces - namespaces.extend([ns.replace(' ', '_') for ns in namespaces if ns and ' ' in ns]) - + namespaces.extend([ns.replace(' ', '_') + for ns in namespaces if ns and ' ' in ns]) return tuple(namespaces) else: return v[0]

def isDefinedNS(self, ns_number): - """Return True if the namespace has been defined in this family. - """ + """Return True if the namespace has been defined in this family.""" + return self.namespaces.has_key(ns_number)

def isNsI18N(self, ns_number, code): """Return True if the namespace has been internationalized. - (it has a custom entry for a given language)""" + + (it has a custom entry for a given language) + + """ return self.namespaces[ns_number].has_key(code)

def isDefinedNSLanguage(self, ns_number, code, fallback='_default'): @@ -2623,8 +2629,8 @@ return self.disambiguationTemplates[fallback] else: raise KeyError( - 'ERROR: title for disambig template in language %s unknown' - % code) +"ERROR: title for disambig template in language %(language_code)s unknown" + % {'language_code': code})

# Returns the title of the special namespace in language 'code', taken from # dictionary above.

Modified: branches/rewrite/pywikibot/login.py =================================================================== --- branches/rewrite/pywikibot/login.py 2008-06-26 21:28:02 UTC (rev 5642) +++ branches/rewrite/pywikibot/login.py 2008-06-26 21:39:01 UTC (rev 5643) @@ -81,32 +81,36 @@ [self.site.family.name][self.site.code] except: raise NoUsername( -u'ERROR: Sysop username for %s:%s is undefined.\nIf you have a sysop account for that site, please add such a line to user-config.py:\n\nsysopnames['%s']['%s'] = 'myUsername'' - % (self.site.family.name, self.site.code, - self.site.family.name, self.site.code)) +u"""ERROR: Sysop username for %(fam_name)s:%(wiki_code)s is undefined. +If you have a sysop account for that site, please add a line to user-config.py: + +sysopnames['%(fam_name)s']['%(wiki_code)s'] = 'myUsername'""" + % {'fam_name': self.site.family.name, + 'wiki_code': self.site.code}) else: try: self.username = config.usernames\ [self.site.family.name][self.site.code] except: raise NoUsername( -u"""ERROR: Username for %s:%s is undefined. +u"""ERROR: Username for %(fam_name)s:%(wiki_code)s is undefined. If you have an account for that site, please add a line to user-config.py:

-usernames['%s']['%s'] = 'myUsername'""" - % (self.site.family.name, self.site.code, - self.site.family.name, self.site.code)) +usernames['%(fam_name)s']['%(wiki_code)s'] = 'myUsername'""" + % {'fam_name': self.site.family.name, + 'wiki_code': self.site.code}) self.password = password if getattr(config, 'password_file', ''): self.readPassword()

def botAllowed(self): - """ - Checks whether the bot is listed on a specific page to comply with + """Check whether the bot is listed on a specific page to comply with the policy on the respective wiki. + """ return True # DEBUG - if botList.has_key(self.site.family.name) and botList[self.site.family.name].has_key(self.site.code): + if botList.has_key(self.site.family.name)\ + and botList[self.site.family.name].has_key(self.site.code): botListPageTitle = botList[self.site.family.name][self.site.code] botListPage = pywikibot.Page(self.site, botListPageTitle) for linkedPage in botListPage.linkedPages(): @@ -117,15 +121,15 @@ # No bot policies on other return True

- def getCookie(self, remember=True, captchaId = None, captchaAnswer = None): - """ - Login to the site. + def getCookie(self, remember=True, captchaId=None, captchaAnswer=None): + """Login to the site.

remember Remember login (default: True) captchaId The id number of the captcha, if any captcha The word displayed in the captcha, if any

Returns cookie data if succesful, None otherwise. + """ predata = { "wpName": self.username.encode(self.site.encoding()), @@ -144,12 +148,16 @@ "User-agent": wikipedia.useragent } data = self.site.urlEncode(predata) - response = urllib2.urlopen(urllib2.Request(self.site.protocol() + '://' + self.site.hostname() + address, data, headers)) + response = urllib2.urlopen( + urllib2.Request(self.site.protocol() + + '://' + self.site.hostname() + + address, data, headers)) data = response.read() wikipedia.cj.save(wikipedia.COOKIEFILE) return "Ok" else: - response, data = self.site.postForm(address, predata, useCookie=False) + response, data = self.site.postForm(address, predata, + useCookie=False) n = 0 Reat=re.compile(': (.*?);') L = [] @@ -182,13 +190,14 @@ raise CaptchaError(id) url = self.site.protocol() + '://' + self.site.hostname() + self.site.captcha_image_address(id) answer = wikipedia.ui.askForCaptcha(url) - return self.getCookie(remember = remember, captchaId = id, captchaAnswer = answer) + return self.getCookie(remember=remember, captchaId=id, + captchaAnswer=answer) else: return None

def storecookiedata(self, data): """ - Stores cookie data. + Store cookie data.

The argument data is the raw data, as returned by getCookie().

@@ -202,30 +211,32 @@ f.close()

def readPassword(self): - """ - Reads passwords from a file. DO NOT FORGET TO REMOVE READ - ACCESS FOR OTHER USERS!!! Use chmod 600 password-file. - All lines below should be valid Python tuples in the form - (code, family, username, password) or (username, password) - to set a default password for an username. Default usernames - should occur above specific usernames. + """Read passwords from a file.

- Example: + DO NOT FORGET TO REMOVE READ ACCESS FOR OTHER USERS!!! Use chmod 600 + password-file. All lines below should be valid Python tuples in the + form (code, family, username, password) or (username, password) to + set a default password for an username. Default usernames should + occur above specific usernames.

- ("my_username", "my_default_password") - ("my_sysop_user", "my_sysop_password") - ("en", "wikipedia", "my_en_user", "my_en_pass") + Example: + + ("my_username", "my_default_password") + ("my_sysop_user", "my_sysop_password") + ("en", "wikipedia", "my_en_user", "my_en_pass") + """ file = open(config.password_file) for line in file: if not line.strip(): continue entry = eval(line) if len(entry) == 2: - if entry[0] == self.username: self.password = entry[1] + if entry[0] == self.username: + self.password = entry[1] elif len(entry) == 4: if entry[0] == self.site.code and \ - entry[1] == self.site.family.name and \ - entry[2] == self.username: + entry[1] == self.site.family.name and \ + entry[2] == self.username: self.password = entry[3] file.close()

@@ -233,18 +244,28 @@ if not self.password: # As we don't want the password to appear on the screen, we set # password = True - self.password = pywikibot.input(u'Password for user %s on %s:' % (self.username, self.site), password = True) + self.password = pywikibot.input( + u'Password for user %(name)s on %(site)s:' + % {'name': self.username, 'site': self.site}, + password = True)

# self.password = self.password.encode(self.site.encoding())

- logger.info(u"Logging in to %s as %s" % (self.site, self.username)) + logger.info(u"Logging in to %(site)s as %(name)s" + % {'name': self.username, 'site': self.site}) cookiedata = self.getCookie() if cookiedata: self.storecookiedata(cookiedata) logger.info(u"Should be logged in now") # Show a warning according to the local bot policy if not self.botAllowed(): - logger.error(u'*** Your username is not listed on [[%s]].\n*** Please make sure you are allowed to use the robot before actually using it!' % botList[self.site.family.name][self.site.code]) + logger.error( + u"Username '%(name)s' is not listed on [[%(page)s]]." + % {'name': self.username, + 'page': botList[self.site.family.name][self.site.code]}) + logger.error( +"Please make sure you are allowed to use the robot before actually using it!") + return False return True else: logger.error(u"Login failed. Wrong password or CAPTCHA answer?") @@ -265,7 +286,8 @@ for arg in wikipedia.handleArgs(): if arg.startswith("-pass"): if len(arg) == 5: - password = pywikibot.input(u'Password for all accounts:', password = True) + password = pywikibot.input(u'Password for all accounts:', + password = True) else: password = arg[6:] elif arg == "-sysop": @@ -286,12 +308,12 @@ for lang in namedict[familyName].iterkeys(): site = pywikibot.getSite(code=lang, fam=familyName) if not forceLogin and site.loggedInAs(sysop = sysop) != None: - logger.info(u'Already logged in on %s' % site) + logger.info(u'Already logged in on %(site)s' % locals()) else: - loginMan = LoginManager(password, sysop = sysop, site = site) + loginMan = LoginManager(password, sysop=sysop, site=site) loginMan.login() else: - loginMan = LoginManager(password, sysop = sysop) + loginMan = LoginManager(password, sysop=sysop) loginMan.login()

if __name__ == "__main__":

Modified: branches/rewrite/pywikibot/throttle.py =================================================================== --- branches/rewrite/pywikibot/throttle.py 2008-06-26 21:28:02 UTC (rev 5642) +++ branches/rewrite/pywikibot/throttle.py 2008-06-26 21:39:01 UTC (rev 5643) @@ -46,8 +46,8 @@ self.last_read = 0 self.last_write = 0 self.next_multiplicity = 1.0 - self.checkdelay = 120 # Check logfile again after this many seconds - self.dropdelay = 360 # Ignore processes that have not made + self.checkdelay = 300 # Check logfile again after this many seconds + self.dropdelay = 750 # Ignore processes that have not made # a check in this many seconds self.releasepid = 1800 # Free the process id after this many seconds self.lastwait = 0.0 @@ -60,7 +60,7 @@ def checkMultiplicity(self): global pid self.lock.acquire() - logger.debug("Checking multiplicity: pid = %s" % pid) + logger.debug("Checking multiplicity: pid = %(pid)s" % globals()) try: processes = [] my_pid = 1 @@ -110,8 +110,8 @@ self.process_multiplicity = count if self.verbosedelay: logger.info( - u"Found %s processes running, including the current process." - % count) +u"Found %(count)s processes running, including the current process." + % locals()) finally: self.lock.release()

@@ -206,10 +206,14 @@

Parameter requestsize is the number of Pages to be read/written; multiply delay time by an appropriate factor. + + Because this seizes the throttle lock, it will prevent any other + thread from writing to the same site until the wait expires. + """ self.lock.acquire() try: - waittime = self.waittime(write=write) + wait = self.waittime(write=write) # Calculate the multiplicity of the next delay based on how # big the request is that is being posted now. # We want to add "one delay" for each factor of two in the @@ -217,13 +221,13 @@ # the delay time for the server. self.next_multiplicity = math.log(1+requestsize)/math.log(2.0) # Announce the delay if it exceeds a preset limit - if waittime > config.noisysleep: - logger.info(u"Sleeping for %.1f seconds, %s" - % (waittime, - time.strftime("%Y-%m-%d %H:%M:%S", - time.localtime())) - ) - time.sleep(waittime) + if wait > config.noisysleep: + logger.info(u"Sleeping for %(wait).1f seconds, %(now)s" + % {'wait': wait, + 'now': time.strftime("%Y-%m-%d %H:%M:%S", + time.localtime()) + } ) + time.sleep(wait) if write: self.last_write = time.time() else: @@ -248,12 +252,12 @@ wait = delay - (time.time() - started) if wait > 0: if wait > config.noisysleep: - logger.warn(u"Sleeping for %.1f seconds, %s" - % (wait, - time.strftime("%Y-%m-%d %H:%M:%S", - time.localtime())) - ) + logger.info(u"Sleeping for %(wait).1f seconds, %(now)s" + % {'wait': wait, + 'now': time.strftime("%Y-%m-%d %H:%M:%S", + time.localtime()) + } ) time.sleep(wait) finally: self.lock.release() - +