jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/416370 ) Change subject: mysql.py: add PyMySql as pure-Python MySQL client library ...................................................................... mysql.py: add PyMySql as pure-Python MySQL client library Add PyMySql as pure-Python MySQL client library. PyMySql will be the preferred library to be imported. Fallback libraries: - MySqlDb: as today - oursql: discontinued Bug: T142021 Bug: T89976 Change-Id: I28725fbe6ea81900c06ca3ccbf02cdc8704fd66a --- M pywikibot/data/mysql.py M requirements.txt M tox.ini 3 files changed, 56 insertions(+), 36 deletions(-) Approvals: Zhuyifei1999: Looks good to me, but someone else must approve Dvorapa: Looks good to me, approved jenkins-bot: Verified diff --git a/pywikibot/data/mysql.py b/pywikibot/data/mysql.py index 02dfd7b..c3757ce 100644 --- a/pywikibot/data/mysql.py +++ b/pywikibot/data/mysql.py @@ -1,27 +1,32 @@ # -*- coding: utf-8 -*- """Miscellaneous helper functions for mysql queries.""" # -# (C) Pywikibot team, 2016-2017 +# (C) Pywikibot team, 2016-2018 # # Distributed under the terms of the MIT license. # from __future__ import absolute_import, unicode_literals -# Requires oursql <https://pythonhosted.org/oursql/> or -# MySQLdb <https://sourceforge.net/projects/mysql-python/> -try: - import oursql as mysqldb -except ImportError: - import MySQLdb as mysqldb - import pywikibot +# Requires PyMySql as first choice or +# MySQLdb <https://sourceforge.net/projects/mysql-python/> +try: + import pymysql as mysqldb +except ImportError: + import MySQLdb as mysqldb + pywikibot.warning('PyMySql not found.') + pywikibot.warning('MySQLdb is deprecated. Use PyMySql instead.') +else: + mysqldb.install_as_MySQLdb() + from pywikibot import config2 as config +from pywikibot.tools import deprecated_args, UnicodeType -def mysql_query(query, params=(), dbname=None, encoding='utf-8', verbose=None): - """ - Yield rows from a MySQL query. +@deprecated_args(encoding=None) +def mysql_query(query, params=None, dbname=None, verbose=None): + """Yield rows from a MySQL query. An example query that yields all ns0 pages might look like:: @@ -31,44 +36,59 @@ FROM page WHERE page_namespace = 0; + From MediaWiki 1.5, all projects use Unicode (UTF-8) character encoding. + Cursor charset is utf8. + @param query: MySQL query to execute - @type query: str + @type query: str (unicode in py2) @param params: input parametes for the query, if needed - @type params: tuple + if list or tuple, %s shall be used as placeholder in the query string. + if a dict, %(key)s shall be used as placeholder in the query string. + @type params: tuple, list or dict of str (unicode in py2) @param dbname: db name @type dbname: str - @param encoding: encoding used by the database - @type encoding: str @param verbose: if True, print query to be executed; if None, config.verbose_output will be used. @type verbose: None or bool @return: generator which yield tuples """ + CHARSET = 'utf8' + + # These are specified in config2.py or user-config.py if verbose is None: verbose = config.verbose_output if config.db_connect_file is None: - conn = mysqldb.connect(config.db_hostname, - db=config.db_name_format.format(dbname), - user=config.db_username, - passwd=config.db_password, - port=config.db_port) + credentials = {'user': config.db_username, + 'passwd': config.db_password} else: - conn = mysqldb.connect(config.db_hostname, - db=config.db_name_format.format(dbname), - read_default_file=config.db_connect_file, - port=config.db_port) + credentials = {'read_default_file': config.db_connect_file} + + conn = mysqldb.connect(config.db_hostname, + db=config.db_name_format.format(dbname), + port=config.db_port, + charset=CHARSET, + **credentials) cursor = conn.cursor() - if verbose: - pywikibot.output('Executing query:\n%s' % query) - query = query.encode(encoding) - params = tuple(p.encode(encoding) for p in params) - if params: - cursor.execute(query, params) - else: - cursor.execute(query) + if verbose: + try: + _query = cursor.mogrify(query, params) + except AttributeError: # if MySQLdb is used. + # Not exactly the same encoding handling as cursor.execute() + # Here it is just for the sake of verbose. + _query = query + if params is not None: + _query = query.format(params) + + if not isinstance(_query, UnicodeType): + _query = UnicodeType(_query, encoding='utf-8') + _query = _query.strip() + _query = '\n'.join(' {0}'.format(l) for l in _query.splitlines()) + pywikibot.output('Executing query:\n%s' % _query) + + cursor.execute(query, params) for row in cursor: yield row diff --git a/requirements.txt b/requirements.txt index e44dee9..75c17c1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -66,10 +66,9 @@ # textlib.py and patrol.py mwparserfromhell>=0.3.3 -# The mysql generator in pagegenerators depends on either oursql or MySQLdb -# pywikibot prefers oursql. Both are Python 2 only; T89976. -oursql ; python_version < '3' -mysqlclient ; python_version >= '3' +# The mysql generator in pagegenerators depends on either PyMySQL or MySQLdb +# pywikibot prefers PyMySQL over MySQLdb (Python 2 only) +PyMySQL # scripts/script_wui.py depends on Lua, which is not available using pip # but can be obtained from: https://github.com/bastibe/lunatic-python diff --git a/tox.ini b/tox.ini index ef0b533..6517474 100644 --- a/tox.ini +++ b/tox.ini @@ -166,6 +166,7 @@ pywikibot/config2.py : N806 pywikibot/cosmetic_changes.py : N803, N806 pywikibot/data/api.py : N803, N806 + pywikibot/data/mysql.py : N806 pywikibot/date.py : E241, N803, N806 pywikibot/diff.py : N806 pywikibot/editor.py : N803, N806 -- To view, visit https://gerrit.wikimedia.org/r/416370 To unsubscribe, or for help writing mail filters, visit https://gerrit.wikimedia.org/r/settings Gerrit-Project: pywikibot/core Gerrit-Branch: master Gerrit-MessageType: merged Gerrit-Change-Id: I28725fbe6ea81900c06ca3ccbf02cdc8704fd66a Gerrit-Change-Number: 416370 Gerrit-PatchSet: 15 Gerrit-Owner: Mpaa &lt;mpaa.wiki(a)gmail.com&gt; Gerrit-Reviewer: Dvorapa &lt;dvorapa(a)seznam.cz&gt; Gerrit-Reviewer: Framawiki &lt;framawiki(a)tools.wmflabs.org&gt; Gerrit-Reviewer: John Vandenberg &lt;jayvdb(a)gmail.com&gt; Gerrit-Reviewer: Mpaa &lt;mpaa.wiki(a)gmail.com&gt; Gerrit-Reviewer: Xqt &lt;info(a)gno.de&gt; Gerrit-Reviewer: Zhuyifei1999 &lt;zhuyifei1999(a)gmail.com&gt; Gerrit-Reviewer: jenkins-bot