jenkins-bot has submitted this change and it was merged.
Change subject: Add OAuth support for Pywikibot ......................................................................
Add OAuth support for Pywikibot
This change depends on mwoauth for OAuth support. A new OAuthLoginManager is used for OAuth tokens retrieval. The whole retrieval process is added to scripts.login module. OAuth tokens are set in config.authenticate and enable automatically if applicable.
Tests are configured by setting OAuth related environment variables. Family names of orain test and beta wiki are changed to adapted to OAuth testing.
Bug: T102602 Change-Id: Ib3b464df3f5f8607dc84ff2d5f2969de4c693561 --- M .travis.yml M pywikibot/comms/http.py M pywikibot/config2.py M pywikibot/data/api.py M pywikibot/login.py M pywikibot/site.py M requirements.txt M scripts/login.py M setup.py M tests/http_tests.py A tests/oauth_tests.py M tox.ini 12 files changed, 354 insertions(+), 14 deletions(-)
Approvals: John Vandenberg: Looks good to me, approved jenkins-bot: Verified
diff --git a/.travis.yml b/.travis.yml index 3e3b93b..dc18b33 100644 --- a/.travis.yml +++ b/.travis.yml @@ -42,7 +42,9 @@ - pip install -r dev-requirements.txt
script: - - if [[ "$PYSETUP_TEST_EXTRAS" != '1' ]]; then pip install requests ; fi + - if [[ "$PYSETUP_TEST_EXTRAS" != '1' ]]; then + pip install requests mwoauth ; + fi
- mkdir ~/.pywikibot
@@ -64,6 +66,15 @@ echo "password_file = os.path.expanduser('~/.pywikibot/passwordfile')" >> ~/.pywikibot/user-config.py ; fi
+ - if [[ -n "$OAUTH_DOMAIN" ]]; then + if [[ -n "$OAUTH_PYWIKIBOT2_USERNAME" ]]; then + printf "usernames['${FAMILY}']['${LANGUAGE}'] = '%q'\n" "$OAUTH_PYWIKIBOT2_USERNAME" >> ~/.pywikibot/user-config.py ; + fi ; + oauth_token_var="OAUTH_TOKENS_${FAMILY^^}_${LANGUAGE^^}" ; + if [[ -n "${!oauth_token_var}" ]]; then + printf "authenticate['${OAUTH_DOMAIN}'] = ('%s')\n" "${!oauth_token_var//:/', '}" >> ~/.pywikibot/user-config.py ; + fi ; + fi - echo "authenticate['wiki.musicbrainz.org'] = ('NOTSPAM', 'NOTSPAM')" >> ~/.pywikibot/user-config.py ;
- echo "max_retries = 2" >> ~/.pywikibot/user-config.py @@ -103,14 +114,16 @@ - python: '2.7' env: LANGUAGE=en FAMILY=wpbeta SITE_ONLY=1 - python: '3.4' - env: LANGUAGE=zh FAMILY=wpbeta SITE_ONLY=1 + env: LANGUAGE=zh FAMILY=wpbeta SITE_ONLY=1 OAUTH_DOMAIN="zh.wikipedia.beta.wmflabs.org" - python: '2.7' env: LANGUAGE=wikia FAMILY=wikia PYWIKIBOT2_TEST_NO_RC=1 - python: '3.3' - env: LANGUAGE=en FAMILY=oraintest SITE_ONLY=1 + env: LANGUAGE=en FAMILY=oraintest SITE_ONLY=1 OAUTH_DOMAIN="test.orain.org" - python: '3.3' env: LANGUAGE=en FAMILY=musicbrainz SITE_ONLY=1 - python: '3.4' + env: LANGUAGE=test FAMILY=wikipedia SITE_ONLY=1 OAUTH_DOMAIN="test.wikipedia.org" + - python: '3.4' env: LANGUAGE=test FAMILY=wikidata SITE_ONLY=1 - python: '3.4' env: LANGUAGE=ar FAMILY=wiktionary PYWIKIBOT2_TEST_NO_RC=1 diff --git a/pywikibot/comms/http.py b/pywikibot/comms/http.py index ff0c4f3..53083a0 100644 --- a/pywikibot/comms/http.py +++ b/pywikibot/comms/http.py @@ -31,6 +31,11 @@
import requests
+try: + import requests_oauthlib +except ImportError as e: + requests_oauthlib = e + if sys.version_info[0] > 2: from http import cookiejar as cookielib from urllib.parse import quote @@ -69,8 +74,6 @@ pywikibot.debug(u"Loading cookies failed.", _logger) else: pywikibot.debug(u"Loaded cookies from file.", _logger) - -session.cookies = cookie_jar
# Prepare flush on quit @@ -257,7 +260,7 @@ for i in range(len(netloc_parts))] for path in netlocs: if path in config.authenticate: - if len(config.authenticate[path]) == 2: + if len(config.authenticate[path]) in [2, 4]: return config.authenticate[path] else: warn('Invalid authentication tokens for %s ' @@ -273,10 +276,20 @@ if PY2 and headers: headers = dict((key, str(value)) for key, value in headers.items()) auth = get_authentication(uri) + if auth is not None and len(auth) == 4: + if isinstance(requests_oauthlib, ImportError): + warn('%s' % requests_oauthlib, ImportWarning) + pywikibot.error('OAuth authentication not supported: %s' + % requests_oauthlib) + auth = None + else: + auth = requests_oauthlib.OAuth1(*auth) + cookies = cookie_jar timeout = config.socket_timeout try: response = session.request(method, uri, data=body, headers=headers, - auth=auth, timeout=timeout, verify=True) + cookies=cookies, auth=auth, timeout=timeout, + verify=True) except Exception as e: http_request.data = e else: diff --git a/pywikibot/config2.py b/pywikibot/config2.py index 0ef584b..d5ba396 100644 --- a/pywikibot/config2.py +++ b/pywikibot/config2.py @@ -162,6 +162,18 @@ # Pywikibot supports wildcard (*) in the prefix of hostname and select the # best match authentication. So you can specify authentication not only for # one site +# +# Pywikibot also support OAuth 1.0a via mwoauth +# https://pypi.python.org/pypi/mwoauth +# +# You can add OAuth tokens to your user-config.py of the following form: +# +# authenticate['en.wikipedia.org'] = ('consumer_key','consumer_secret', +# 'access_key', 'access_secret') +# authenticate['*.wikipedia.org'] = ('consumer_key','consumer_secret', +# 'access_key', 'access_secret') +# +# Note: the target wiki site must install OAuth extension authenticate = {}
# diff --git a/pywikibot/data/api.py b/pywikibot/data/api.py index 7d08ea0..c50e46d 100644 --- a/pywikibot/data/api.py +++ b/pywikibot/data/api.py @@ -31,7 +31,7 @@ from pywikibot import config, login from pywikibot.tools import MediaWikiVersion, deprecated, itergroup, ip, PY2 from pywikibot.exceptions import ( - Server504Error, Server414Error, FatalServerError, Error + Server504Error, Server414Error, FatalServerError, NoUsername, Error ) from pywikibot.comms import http
@@ -1867,7 +1867,8 @@ """ self._add_defaults() if (not config.enable_GET_without_SSL and - self.site.protocol() != 'https'): + self.site.protocol() != 'https' or + self.site.is_oauth_token_available()): # work around T108182 use_get = False elif self.use_get is None: if self.action == 'query': @@ -2093,6 +2094,9 @@ self.site.user(), ', '.join('{0}: {1}'.format(*e) for e in user_tokens.items()))) + if 'mwoauth-invalid-authorization' in code: + raise NoUsername('Failed OAuth authentication for %s: %s' + % (self.site, info)) # raise error try: # Due to bug T66958, Page's repr may return non ASCII bytes diff --git a/pywikibot/login.py b/pywikibot/login.py index 7c0bcef..ac37803 100644 --- a/pywikibot/login.py +++ b/pywikibot/login.py @@ -14,14 +14,25 @@ import codecs import os import stat +import webbrowser
from warnings import warn + +try: + import mwoauth +except ImportError as e: + mwoauth = e
import pywikibot
from pywikibot import config from pywikibot.tools import deprecated_args, normalize_username from pywikibot.exceptions import NoUsername + + +class OAuthImpossible(ImportError): + + """OAuth authentication is not possible on your system."""
class _PasswordFileWarning(UserWarning): @@ -296,3 +307,119 @@ def showCaptchaWindow(self, url): """Open a window to show the captcha for the given URL.""" pass + + +class OauthLoginManager(LoginManager): + + """Site login manager using OAuth.""" + + # NOTE: Currently OauthLoginManager use mwoauth directly to complete OAuth + # authentication process + + def __init__(self, password=None, sysop=False, site=None, user=None): + """ + Constructor. + + All parameters default to defaults in user-config. + + @param site: Site object to log into + @type site: BaseSite + @param user: consumer key + @type user: basestring + @param password: consumer secret + @type password: basestring + @param sysop: login as sysop account. + The sysop username is loaded from config.sysopnames. + @type sysop: bool + + @raises NoUsername: No username is configured for the requested site. + @raise OAuthImpossible: mwoauth isn't installed + """ + if isinstance(mwoauth, ImportError): + raise OAuthImpossible('mwoauth is not installed: %s.' % mwoauth) + assert password is not None and user is not None + assert sysop is False + super(OauthLoginManager, self).__init__(None, False, site, None) + if self.password: + pywikibot.warn('Password exists in password file for %s:%s.' + 'Password is unnecessary and should be removed ' + 'when OAuth enabled.' % (self.site, self.username)) + self._consumer_token = (user, password) + self._access_token = None + + def login(self, retry=False, force=False): + """ + Attempt to log into the server. + + @param retry: infinitely retry if exception occurs during authentication. + @type retry: bool + @param force: force to re-authenticate + @type force: bool + """ + if self.access_token is None or force: + pywikibot.output('Logging in to %(site)s via OAuth consumer %(key)s' + % {'key': self.consumer_token[0], + 'site': self.site}) + consumer_token = mwoauth.ConsumerToken(self.consumer_token[0], + self.consumer_token[1]) + handshaker = mwoauth.Handshaker( + self.site.base_url(self.site.path()), consumer_token) + try: + redirect, request_token = handshaker.initiate() + pywikibot.stdout('Authenticate via web browser..') + webbrowser.open(redirect) + pywikibot.stdout('If your web browser does not open ' + 'automatically, please point it to: %s' + % redirect) + request_qs = pywikibot.input('Response query string: ') + access_token = handshaker.complete(request_token, + request_qs) + self._access_token = (access_token.key, access_token.secret) + except Exception as e: + pywikibot.error(e) + if retry: + self.login(retry=True, force=force) + else: + pywikibot.output('Logged in to %(site)s via consumer %(key)s' + % {'key': self.consumer_token[0], + 'site': self.site}) + + @property + def consumer_token(self): + """ + OAuth consumer key token and secret token. + + @rtype: tuple of two str + """ + return self._consumer_token + + @property + def access_token(self): + """ + OAuth access key token and secret token. + + @rtype: tuple of two str + """ + return self._access_token + + @property + def identity(self): + """ + Get identifying information about a user via an authorized token. + + @rtype: None or dict + """ + if self.access_token is None: + pywikibot.error('Access token not set') + return None + consumer_token = mwoauth.ConsumerToken(self.consumer_token[0], + self.consumer_token[1]) + access_token = mwoauth.AccessToken(self.access_token[0], + self.access_token[1]) + try: + identity = mwoauth.identify(self.site.base_url(self.site.path()), + consumer_token, access_token) + return identity + except Exception as e: + pywikibot.error(e) + return None diff --git a/pywikibot/site.py b/pywikibot/site.py index d750bcb..4e87cb6 100644 --- a/pywikibot/site.py +++ b/pywikibot/site.py @@ -38,6 +38,7 @@ manage_wrapping, MediaWikiVersion, first_upper, normalize_username, merge_unique_dicts, ) +from pywikibot.comms.http import get_authentication from pywikibot.tools.ip import is_IP from pywikibot.throttle import Throttle from pywikibot.data import api @@ -1789,6 +1790,15 @@ """ return self.logged_in(sysop) and self.user()
+ def is_oauth_token_available(self): + """ + Check whether OAuth token is set for this site. + + @rtype: bool + """ + auth_token = get_authentication(self.base_url('')) + return auth_token is not None and len(auth_token) == 4 + def login(self, sysop=False): """Log the user in if not already logged in.""" # TODO: this should include an assert that loginstatus @@ -1812,6 +1822,7 @@ if sysop else LoginStatus.AS_USER) return # check whether a login cookie already exists for this user + # or check user identity when OAuth enabled self._loginstatus = LoginStatus.IN_PROGRESS try: self.getuserinfo(force=True) @@ -1820,6 +1831,17 @@ return except api.APIError: # May occur if you are not logged in (no API read permissions). pass + if self.is_oauth_token_available(): + if sysop: + raise NoUsername('No sysop is permitted with OAuth') + elif self.userinfo['name'] != self._username[sysop]: + raise NoUsername('Logged in on %(site)s via OAuth as %(wrong)s, ' + 'but expect as %(right)s' + % {'site': self, + 'wrong': self.userinfo['name'], + 'right': self._username[sysop]}) + else: + raise NoUsername('Logging in on %s via OAuth failed' % self) loginMan = api.LoginManager(site=self, sysop=sysop, user=self._username[sysop]) if loginMan.login(retry=True): @@ -1838,7 +1860,11 @@ """Logout of the site and load details for the logged out user.
Also logs out of the global account if linked to the user. + + @raise APIError: Logout is not available when OAuth enabled. """ + if self.is_oauth_token_available(): + pywikibot.warning('Using OAuth suppresses logout function') uirequest = self._simple_request(action='logout') uirequest.submit() self._loginstatus = LoginStatus.NOT_LOGGED_IN diff --git a/requirements.txt b/requirements.txt index da61fb8..f10734a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,6 +36,11 @@
unicodedata2>=7.0.0-2 ; python_version < '3'
+# OAuth support +# mwoauth 0.2.4 is needed because it supports getting identity information +# about the user +mwoauth>=0.2.4 + # core interwiki_graph.py: git+https://github.com/nlhepler/pydot#egg=pydot-1.0.29
diff --git a/scripts/login.py b/scripts/login.py index 89a5fa7..5d8c3ea 100755 --- a/scripts/login.py +++ b/scripts/login.py @@ -35,6 +35,11 @@
-sysop Log in with your sysop account.
+ -oauth Generate OAuth authentication information. + NOTE: Need to copy OAuth tokens to your user-config.py + manually. -logout, -pass, -force, -pass:XXXX and -sysop are not + compatible with -oauth. + If not given as parameter, the script will ask for your username and password (password entry will be hidden), log in to your home wiki using this combination, and store the resulting cookies (containing your password @@ -60,7 +65,45 @@ import pywikibot from os.path import join from pywikibot import config +from pywikibot.login import OauthLoginManager from pywikibot.exceptions import SiteDefinitionError + + +def _get_consumer_token(site): + key_msg = 'OAuth consumer key on {0}:{1}'.format(site.code, site.family) + key = pywikibot.input(key_msg) + secret_msg = 'OAuth consumer secret for consumer {0}'.format(key) + secret = pywikibot.input(secret_msg, password=True) + return key, secret + + +def _oauth_login(site): + consumer_key, consumer_secret = _get_consumer_token(site) + login_manager = OauthLoginManager(consumer_secret, False, site, + consumer_key) + login_manager.login() + identity = login_manager.identity + if identity is None: + pywikibot.error('Invalid OAuth info for %(site)s.' % + {'site': site}) + elif site.username() != identity['username']: + pywikibot.error('Logged in on %(site)s via OAuth as %(wrong)s, ' + 'but expect as %(right)s' + % {'site': site, + 'wrong': identity['username'], + 'right': site.username()}) + else: + oauth_token = login_manager.consumer_token + login_manager.access_token + pywikibot.output('Logged in on %(site)s as %(username)s' + 'via OAuth consumer %(consumer)s' + % {'site': site, + 'username': site.username(sysop=False), + 'consumer': consumer_key}) + pywikibot.output('NOTE: To use OAuth, you need to copy the ' + 'following line to your user-config.py:') + pywikibot.output('authenticate['%(hostname)s'] = %(oauth_token)s' % + {'hostname': site.hostname(), + 'oauth_token': oauth_token})
def main(*args): @@ -76,6 +119,7 @@ sysop = False logall = False logout = False + oauth = False unknown_args = [] for arg in pywikibot.handle_args(args): if arg.startswith("-pass"): @@ -95,6 +139,8 @@ join(config.base_dir, 'pywikibot.lwp')) elif arg == "-logout": logout = True + elif arg == '-oauth': + oauth = True else: unknown_args += [arg]
@@ -103,7 +149,7 @@ return False
if logall: - if sysop: + if sysop and not oauth: namedict = config.sysopnames else: namedict = config.usernames @@ -114,6 +160,9 @@ for lang in namedict[familyName]: try: site = pywikibot.Site(code=lang, fam=familyName) + if oauth: + _oauth_login(site) + continue if logout: site.logout() else: diff --git a/setup.py b/setup.py index f541fa3..9e9ed5c 100644 --- a/setup.py +++ b/setup.py @@ -51,6 +51,7 @@ # 0.6.1 supports socket.io 1.0, but WMF is using 0.9 (T91393 and T85716) 'rcstream': ['socketIO-client<0.6.1'], 'security': ['requests[security]'], + 'mwoauth': ['mwoauth>=0.2.4'], }
if PY2: diff --git a/tests/http_tests.py b/tests/http_tests.py index b760e79..6874b17 100644 --- a/tests/http_tests.py +++ b/tests/http_tests.py @@ -97,9 +97,9 @@ self._authenticate = config.authenticate config.authenticate = { 'zh.wikipedia.beta.wmflabs.org': ('1', '2'), - '*.wikipedia.beta.wmflabs.org': ('3', '4'), + '*.wikipedia.beta.wmflabs.org': ('3', '4', '3', '4'), '*.beta.wmflabs.org': ('5', '6'), - '*.wmflabs.org': ('7', '8'), + '*.wmflabs.org': ('7', '8', '8'), }
def tearDown(self): @@ -110,9 +110,9 @@ """Test url-based authentication info.""" pairs = { 'https://zh.wikipedia.beta.wmflabs.org': ('1', '2'), - 'https://en.wikipedia.beta.wmflabs.org': ('3', '4'), + 'https://en.wikipedia.beta.wmflabs.org': ('3', '4', '3', '4'), 'https://wiki.beta.wmflabs.org': ('5', '6'), - 'https://beta.wmflabs.org': ('7', '8'), + 'https://beta.wmflabs.org': None, 'https://wmflabs.org': None, 'https://www.wikiquote.org/': None, } diff --git a/tests/oauth_tests.py b/tests/oauth_tests.py new file mode 100644 index 0000000..111c0d8 --- /dev/null +++ b/tests/oauth_tests.py @@ -0,0 +1,89 @@ +# -*- coding: utf-8 -*- +"""Test OAuth functionality.""" +# +# (C) Pywikibot team, 2015 +# +# Distributed under the terms of the MIT license. +# +from __future__ import unicode_literals + +__version__ = '$Id$' + +import os + +try: + import mwoauth +except ImportError as e: + mwoauth = e + +from pywikibot.login import OauthLoginManager +from tests.aspects import ( + unittest, + DefaultSiteTestCase, +) + + +class OAuthSiteTestCase(DefaultSiteTestCase): + + """Run tests related to OAuth authentication.""" + + @classmethod + def setUpClass(cls): + """Check if mwoauth is installed.""" + super(OAuthSiteTestCase, cls).setUpClass() + if isinstance(mwoauth, ImportError): + raise unittest.SkipTest('mwoauth not installed') + + def _get_oauth_tokens(self): + """Get valid OAuth tokens from environment variables.""" + tokens_env = 'OAUTH_TOKENS_' + self.family.upper() + tokens = os.environ.get(tokens_env + '_' + self.code.upper(), None) + tokens = tokens or os.environ.get(tokens_env, None) + return tuple(tokens.split(':')) if tokens is not None else None + + def setUp(self): + """Check if OAuth extension is installed and OAuth tokens are set.""" + super(OAuthSiteTestCase, self).setUp() + self.site = self.get_site() + if not self.site.has_extension('OAuth'): + raise unittest.SkipTest('OAuth extension not loaded on test site') + tokens = self._get_oauth_tokens() + if tokens is None: + raise unittest.SkipTest('OAuth tokens not set') + self.assertEqual(len(tokens), 4) + self.consumer_token = tokens[:2] + self.access_token = tokens[2:] + + +class TestOauthLoginManger(OAuthSiteTestCase): + + """Test OAuth login manager.""" + + def _get_login_manager(self): + login_manager = OauthLoginManager(self.consumer_token[1], False, + self.site, self.consumer_token[0]) + # Set access token directly, discard user interaction token fetching + login_manager._access_token = self.access_token + return login_manager + + def test_login(self): + """Test login.""" + login_manager = self._get_login_manager() + login_manager.login() + self.assertEqual(login_manager.consumer_token, self.consumer_token) + self.assertEqual(login_manager.access_token, self.access_token) + + def test_identity(self): + """Test identity.""" + login_manager = self._get_login_manager() + self.assertIsNotNone(login_manager.access_token) + self.assertIsInstance(login_manager.identity, dict) + self.assertEqual(login_manager.identity['username'], + self.site.username(sysop=False)) + + +if __name__ == '__main__': + try: + unittest.main() + except SystemExit: + pass diff --git a/tox.ini b/tox.ini index c85de9f..6b9c5cf 100644 --- a/tox.ini +++ b/tox.ini @@ -138,6 +138,7 @@ tests/logentry_tests.py \ tests/mediawikiversion_tests.py \ tests/namespace_tests.py \ + tests/oauth_tests.py \ tests/proofreadpage_tests.py \ tests/protectbot_tests.py \ tests/pwb/ \