jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/416370 )

Change subject: mysql.py: add PyMySql as pure-Python MySQL client library ......................................................................

mysql.py: add PyMySql as pure-Python MySQL client library

Add PyMySql as pure-Python MySQL client library. PyMySql will be the preferred library to be imported. Fallback libraries: - MySqlDb: as today - oursql: discontinued

Bug: T142021 Bug: T89976 Change-Id: I28725fbe6ea81900c06ca3ccbf02cdc8704fd66a --- M pywikibot/data/mysql.py M requirements.txt M tox.ini 3 files changed, 56 insertions(+), 36 deletions(-)

Approvals: Zhuyifei1999: Looks good to me, but someone else must approve Dvorapa: Looks good to me, approved jenkins-bot: Verified

diff --git a/pywikibot/data/mysql.py b/pywikibot/data/mysql.py index 02dfd7b..c3757ce 100644 --- a/pywikibot/data/mysql.py +++ b/pywikibot/data/mysql.py @@ -1,27 +1,32 @@ # -*- coding: utf-8 -*- """Miscellaneous helper functions for mysql queries.""" # -# (C) Pywikibot team, 2016-2017 +# (C) Pywikibot team, 2016-2018 # # Distributed under the terms of the MIT license. # from __future__ import absolute_import, unicode_literals

-# Requires oursql https://pythonhosted.org/oursql/ or -# MySQLdb https://sourceforge.net/projects/mysql-python/ -try: - import oursql as mysqldb -except ImportError: - import MySQLdb as mysqldb - import pywikibot

+# Requires PyMySql as first choice or +# MySQLdb https://sourceforge.net/projects/mysql-python/ +try: + import pymysql as mysqldb +except ImportError: + import MySQLdb as mysqldb + pywikibot.warning('PyMySql not found.') + pywikibot.warning('MySQLdb is deprecated. Use PyMySql instead.') +else: + mysqldb.install_as_MySQLdb() + from pywikibot import config2 as config +from pywikibot.tools import deprecated_args, UnicodeType

-def mysql_query(query, params=(), dbname=None, encoding='utf-8', verbose=None): - """ - Yield rows from a MySQL query. +@deprecated_args(encoding=None) +def mysql_query(query, params=None, dbname=None, verbose=None): + """Yield rows from a MySQL query.

An example query that yields all ns0 pages might look like::

@@ -31,44 +36,59 @@ FROM page WHERE page_namespace = 0;

+ From MediaWiki 1.5, all projects use Unicode (UTF-8) character encoding. + Cursor charset is utf8. + @param query: MySQL query to execute - @type query: str + @type query: str (unicode in py2) @param params: input parametes for the query, if needed - @type params: tuple + if list or tuple, %s shall be used as placeholder in the query string. + if a dict, %(key)s shall be used as placeholder in the query string. + @type params: tuple, list or dict of str (unicode in py2) @param dbname: db name @type dbname: str - @param encoding: encoding used by the database - @type encoding: str @param verbose: if True, print query to be executed; if None, config.verbose_output will be used. @type verbose: None or bool @return: generator which yield tuples """ + CHARSET = 'utf8' + + # These are specified in config2.py or user-config.py if verbose is None: verbose = config.verbose_output

if config.db_connect_file is None: - conn = mysqldb.connect(config.db_hostname, - db=config.db_name_format.format(dbname), - user=config.db_username, - passwd=config.db_password, - port=config.db_port) + credentials = {'user': config.db_username, + 'passwd': config.db_password} else: - conn = mysqldb.connect(config.db_hostname, - db=config.db_name_format.format(dbname), - read_default_file=config.db_connect_file, - port=config.db_port) + credentials = {'read_default_file': config.db_connect_file} + + conn = mysqldb.connect(config.db_hostname, + db=config.db_name_format.format(dbname), + port=config.db_port, + charset=CHARSET, + **credentials)

cursor = conn.cursor() - if verbose: - pywikibot.output('Executing query:\n%s' % query) - query = query.encode(encoding) - params = tuple(p.encode(encoding) for p in params)

- if params: - cursor.execute(query, params) - else: - cursor.execute(query) + if verbose: + try: + _query = cursor.mogrify(query, params) + except AttributeError: # if MySQLdb is used. + # Not exactly the same encoding handling as cursor.execute() + # Here it is just for the sake of verbose. + _query = query + if params is not None: + _query = query.format(params) + + if not isinstance(_query, UnicodeType): + _query = UnicodeType(_query, encoding='utf-8') + _query = _query.strip() + _query = '\n'.join(' {0}'.format(l) for l in _query.splitlines()) + pywikibot.output('Executing query:\n%s' % _query) + + cursor.execute(query, params)

for row in cursor: yield row diff --git a/requirements.txt b/requirements.txt index e44dee9..75c17c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -66,10 +66,9 @@ # textlib.py and patrol.py mwparserfromhell>=0.3.3

-# The mysql generator in pagegenerators depends on either oursql or MySQLdb -# pywikibot prefers oursql. Both are Python 2 only; T89976. -oursql ; python_version < '3' -mysqlclient ; python_version >= '3' +# The mysql generator in pagegenerators depends on either PyMySQL or MySQLdb +# pywikibot prefers PyMySQL over MySQLdb (Python 2 only) +PyMySQL

# scripts/script_wui.py depends on Lua, which is not available using pip # but can be obtained from: https://github.com/bastibe/lunatic-python diff --git a/tox.ini b/tox.ini index ef0b533..6517474 100644 --- a/tox.ini +++ b/tox.ini @@ -166,6 +166,7 @@ pywikibot/config2.py : N806 pywikibot/cosmetic_changes.py : N803, N806 pywikibot/data/api.py : N803, N806 + pywikibot/data/mysql.py : N806 pywikibot/date.py : E241, N803, N806 pywikibot/diff.py : N806 pywikibot/editor.py : N803, N806