Revision: 6171 Author: russblau Date: 2008-12-19 18:16:50 +0000 (Fri, 19 Dec 2008)

Log Message: ----------- More output formatting, and login bug fixes

Modified Paths: -------------- branches/rewrite/pywikibot/__init__.py branches/rewrite/pywikibot/bot.py branches/rewrite/pywikibot/comms/http.py branches/rewrite/pywikibot/data/api.py branches/rewrite/pywikibot/login.py

Modified: branches/rewrite/pywikibot/__init__.py =================================================================== --- branches/rewrite/pywikibot/__init__.py 2008-12-19 12:30:48 UTC (rev 6170) +++ branches/rewrite/pywikibot/__init__.py 2008-12-19 18:16:50 UTC (rev 6171) @@ -18,7 +18,9 @@ import textlib from bot import *

+logging.basicConfig(fmt="%(message)s")

+ def deprecate_arg(old_arg, new_arg): """Decorator to declare old_arg deprecated and replace it with new_arg""" logger = logging.getLogger() @@ -102,30 +104,6 @@ link_regex = re.compile(r'[[(?P<title>[^]|[#<>{}]*)(|.*?)?]]')

-# User interface functions (kept extremely simple for debugging) - -def output(text, toStdout=False): - if toStdout: - level = STDOUT - else: - level = logging.INFO - logging.getLogger().log(level, - text.encode(config.console_encoding, - "xmlcharrefreplace")) - -def input(prompt, password=False): - if isinstance(prompt, unicode): - encoding = sys.stdout.encoding - # Fallback to _some_ encoding for virtual consoles (cron, screen !) - if not encoding: - encoding = "UTF-8" - prompt = prompt.encode(encoding, "xmlcharrefreplace") - if password: - import getpass - return getpass.getpass(prompt) - return raw_input(prompt) - - def set_debug(layer): """Set the logger for specified layer to DEBUG level.

@@ -144,7 +122,7 @@ This method does not check the 'layer' argument for validity.

""" - logging.getLogger(layer).setLevel(logging.DEBUG) + logging.getLogger(layer).setLevel(DEBUG)

# Throttle and thread handling @@ -175,7 +153,7 @@ # only need one drop() call because all throttles use the same global pid try: _sites[_sites.keys()[0]].throttle.drop() - logger.log(pywikibot.VERBOSE, "Dropped throttle(s).") + logger.log(VERBOSE, "Dropped throttle(s).") except IndexError: pass

Modified: branches/rewrite/pywikibot/bot.py =================================================================== --- branches/rewrite/pywikibot/bot.py 2008-12-19 12:30:48 UTC (rev 6170) +++ branches/rewrite/pywikibot/bot.py 2008-12-19 18:16:50 UTC (rev 6171) @@ -23,11 +23,26 @@

# logging levels

+from logging import DEBUG, INFO, WARNING, ERROR, CRITICAL STDOUT = 16 VERBOSE = 18 INPUT = 25

+def output(text, decoder=None, newline=True, toStdout=False, level=INFO): + if toStdout: + level = STDOUT + logging.getLogger().log(level, text) + + +def input(prompt, password=False): + logging.getLogger().log(INPUT, prompt) + if password: + import getpass + return getpass.getpass("") + return raw_input() + + def calledModuleName(): """Return the name of the module calling this function.

@@ -42,14 +57,19 @@ return os.path.basename(called)

-class LevelFilter(logging.Filter): - """Filter that only passes records at a specific level.""" +class MaxLevelFilter(logging.Filter): + """Filter that only passes records at or below a specific level. + + (setting handler level only passes records at or *above* a specified level, + so this provides the opposite functionality) + + """ def __init__(self, level=None): self.level = level

def filter(self, record): if self.level: - return record.levelno == self.level + return record.levelno <= self.level else: return True

@@ -93,6 +113,7 @@ if not moduleName: moduleName = "terminal-interface" nonGlobalArgs = [] + username = None for arg in args: arg = _decodeArg(arg) if arg == '-help': @@ -101,7 +122,9 @@ elif arg.startswith('-family:'): config.family = arg[8:] elif arg.startswith('-lang:'): - config.code = arg[6:] + config.mylang = arg[6:] + elif arg.startswith("-user:"): + username = arg[6:] elif arg.startswith('-putthrottle:'): config.put_throttle = int(arg[13:]) elif arg.startswith('-pt:'): @@ -133,8 +156,29 @@ # about it. nonGlobalArgs.append(arg)

+ if username: + config.usernames[config.family][config.mylang] = username + # initialize logging system for terminal-based bots

+ # All user output is routed through the logging module. + # Each type of output is handled by an appropriate handler object. + # This structure is used to permit eventual development of other + # user interfaces (GUIs) without modifying the core bot code. + # The following output levels are defined: + # DEBUG - only for file logging; debugging messages + # STDOUT - output that must be sent to sys.stdout (for bots that may + # have their output redirected to a file or other destination) + # VERBOSE - optional progress information for display to user + # INFO - normal (non-optional) progress information for display to user + # INPUT - prompts requiring user response + # WARN - user warning messages + # ERROR - user error messages + # CRITICAL - fatal error messages + # Accordingly, do ''not'' use print statements in bot code; instead, + # send output to the pywikibot.output() function which will route it + # to the logging module. + logging.addLevelName(VERBOSE, "VERBOSE") # for messages to be displayed on terminal at "verbose" setting # use INFO for messages to be displayed even on non-verbose setting @@ -143,15 +187,20 @@ logging.addLevelName(INPUT, "INPUT") # for prompts requiring user response

- logging.basicConfig(format="%(message)s") # initialize root logger root_logger = logging.getLogger() + # default handler for VERBOSE and INFO levels default_handler = root_logger.handlers[0] - root_logger.setLevel(logging.DEBUG) # all records go to logger - # handlers filter separately by level + root_logger.setLevel(DEBUG) # all records go to logger + + # configure default handler for VERBOSE, INFO, and INPUT levels if config.verbose_output: default_handler.setLevel(VERBOSE) else: - default_handler.setLevel(logging.INFO) + default_handler.setLevel(INFO) + default_handler.addFilter(MaxLevelFilter(INPUT)) + default_handler.setFormatter(logging.Formatter(fmt="%(message)s")) + + # if user has enabled file logging, configure file handler if moduleName in config.log or '*' in config.log: if config.logfilename: logfile = config.datafilepath(config.logfilename) @@ -160,7 +209,7 @@ file_handler = logging.handlers.RotatingFileHandler( filename=logfile, maxBytes=2 << 20, backupCount=5) if config.debug_log: - file_handler.setLevel(logging.DEBUG) + file_handler.setLevel(DEBUG) else: file_handler.setLevel(VERBOSE) form = logging.Formatter( @@ -171,11 +220,19 @@ file_handler.setFormatter(form) root_logger.addHandler(file_handler)

+ # handler for level STDOUT output_handler = logging.StreamHandler(strm=sys.stdout) output_handler.setLevel(STDOUT) - output_handler.addFilter(LevelFilter(STDOUT)) + output_handler.addFilter(MaxLevelFilter(STDOUT)) root_logger.addHandler(output_handler)

+ # handler for levels WARNING and higher + warning_handler = logging.StreamHandler() # uses sys.stderr + warning_handler.setLevel(logging.WARNING) + warning_handler.setFormatter( + logging.Formatter(fmt="%(levelname)s: %(message)s")) + root_logger.addHandler(warning_handler) + if config.verbose_output: import re ver = pywikibot.__version__ # probably can be improved on @@ -183,6 +240,7 @@ pywikibot.output(u'Pywikipediabot r%s' % m.group(1)) pywikibot.output(u'Python %s' % sys.version)

+ root_logger.debug("handleArgs() completed.") return nonGlobalArgs

@@ -203,11 +261,13 @@ wikipedia, wiktionary, wikitravel, ... This will override the configuration in user-config.py.

--daemonize:xyz Immediately returns control to the terminal and redirects +-user:xyz Log in as user 'xyz' instead of the default username. + +-daemonize:xyz Immediately return control to the terminal and redirect stdout and stderr to xyz (only use for bots that require no input from stdin).

--help Shows this help text. +-help Show this help text.

-log Enable the logfile, using the default filename '%s-bot.log'

Modified: branches/rewrite/pywikibot/comms/http.py =================================================================== --- branches/rewrite/pywikibot/comms/http.py 2008-12-19 12:30:48 UTC (rev 6170) +++ branches/rewrite/pywikibot/comms/http.py 2008-12-19 18:16:50 UTC (rev 6171) @@ -44,10 +44,7 @@ http_queue = Queue.Queue()

cookie_jar = threadedhttp.LockableCookieJar( - config.datafilepath("%s-%s-%s.lwp" - % (config.family, - config.mylang, - config.usernames[config.family][config.mylang]))) + config.datafilepath("pywikibot.lwp")) try: cookie_jar.load() except (IOError, cookielib.LoadError):

Modified: branches/rewrite/pywikibot/data/api.py =================================================================== --- branches/rewrite/pywikibot/data/api.py 2008-12-19 12:30:48 UTC (rev 6170) +++ branches/rewrite/pywikibot/data/api.py 2008-12-19 18:16:50 UTC (rev 6171) @@ -23,7 +23,7 @@ from pywikibot import login from pywikibot.exceptions import *

-logger = logging.getLogger("data") +logger = logging.getLogger()

lagpattern = re.compile(r"Waiting for [\d.]+: (?P<lag>\d+) seconds? lagged")

@@ -198,6 +198,7 @@ logger.warning("%s, %s", uri, params) self.wait() continue + logger.debug("API response received:\n%s", rawdata) if not isinstance(rawdata, unicode): rawdata = rawdata.decode(self.site.encoding()) if rawdata.startswith(u"unknown_action"): @@ -228,11 +229,11 @@ self.site._userinfo = result['query']['userinfo']

if "warnings" in result: - modules = [k for k in result["warning"] if k != "info"] - logger.warn( - "API warning (%s): %s" - % (", ".join(modules), result['warnings']['info'])) - warnings.warn(result['warnings']['info']) + modules = [k for k in result["warnings"] if k != "info"] + for mod in modules: + logger.warning( + "API warning (%s): %s" + % (mod, result["warnings"][mod]["*"])) if "error" not in result: return result if "*" in result["error"]: @@ -558,19 +559,22 @@ login_result = login_request.submit() if u"login" not in login_result: raise RuntimeError("API login response does not have 'login' key.") - if login_result['login']['result'] != u'Success': - self._waituntil = datetime.datetime.now() + datetime.timedelta(seconds=60) - return None + if login_result['login']['result'] == u'Success': + prefix = login_result['login']['cookieprefix'] + cookies = [] + for key in ('Token', 'UserID', 'UserName'): + cookies.append("%s%s=%s" + % (prefix, key, + login_result['login']['lg'+key.lower()])) + self.username = login_result['login']['lgusername'] + return "\n".join(cookies) + elif login_result['login']['result'] == "Throttled": + self._waituntil = datetime.now() \ + + timedelta(seconds=int( + login_result["login"]["wait"]) + ) + raise APIError(code=login_result["login"]["result"], info="")

- prefix = login_result['login']['cookieprefix'] - cookies = [] - for key in ('Token', 'UserID', 'UserName'): - cookies.append("%s%s=%s" - % (prefix, key, - login_result['login']['lg'+key.lower()])) - self.username = login_result['login']['lgusername'] - return "\n".join(cookies) - def storecookiedata(self, data): pywikibot.cookie_jar.save()

Modified: branches/rewrite/pywikibot/login.py =================================================================== --- branches/rewrite/pywikibot/login.py 2008-12-19 12:30:48 UTC (rev 6170) +++ branches/rewrite/pywikibot/login.py 2008-12-19 18:16:50 UTC (rev 6171) @@ -50,7 +50,7 @@ from pywikibot import config from pywikibot.exceptions import *

-logger = logging.getLogger("wiki") +logger = logging.getLogger()

# On some wikis you are only allowed to run a bot if there is a link to @@ -191,27 +191,29 @@

logger.info(u"Logging in to %(site)s as %(name)s" % {'name': self.username, 'site': self.site}) - cookiedata = self.getCookie() - if cookiedata: - self.storecookiedata(cookiedata) - logger.info(u"Should be logged in now") - # Show a warning according to the local bot policy - if not self.botAllowed(): - logger.error( - u"Username '%(name)s' is not listed on [[%(page)s]]." - % {'name': self.username, - 'page': botList[self.site.family.name][self.site.code]}) - logger.error( -"Please make sure you are allowed to use the robot before actually using it!") - return False - return True - else: - logger.error(u"Login failed. Wrong password or CAPTCHA answer?") + try: + cookiedata = self.getCookie() + except pywikibot.data.api.APIError, e: + logger.error("Login failed (%s).", e.code) if retry: self.password = None return self.login(retry = True) else: return False + self.storecookiedata(cookiedata) + logger.info(u"Should be logged in now") +## # Show a warning according to the local bot policy +## FIXME: disabled due to recursion; need to move this to the Site object after +## login +## if not self.botAllowed(): +## logger.error( +## u"Username '%(name)s' is not listed on [[%(page)s]]." +## % {'name': self.username, +## 'page': botList[self.site.family.name][self.site.code]}) +## logger.error( +##"Please make sure you are allowed to use the robot before actually using it!") +## return False + return True

def showCaptchaWindow(self, url): pass @@ -221,7 +223,7 @@ sysop = False logall = False forceLogin = False - for arg in wikipedia.handleArgs(): #FIXME + for arg in pywikibot.handleArgs(): if arg.startswith("-pass"): if len(arg) == 5: password = pywikibot.input(u'Password for all accounts:', @@ -235,7 +237,7 @@ elif arg == "-force": forceLogin = True else: - wikipedia.showHelp('login') #FIXME + pywikibot.showHelp('login') return if logall: if sysop: @@ -257,7 +259,7 @@ u' is not a valid site, please remove it from your config')

else: - loginMan = LoginManager(password, sysop=sysop) + loginMan = pywikibot.data.api.LoginManager(password, sysop=sysop) loginMan.login()

if __name__ == "__main__":