jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/272990 )
Change subject: [IMPR] Add documentation for sql parameter in replace.py ......................................................................
[IMPR] Add documentation for sql parameter in replace.py
+ also unify -sql parameters to -mysqlquery between scripts + move ImportError from scripts/table2wiki.py to pywikibot/data/mysql.py + improve documentation + narrow database replica hostname for Toolforge
Bug: T124869 Bug: T182523 Change-Id: I257edc61c6919542227e4b8c47799af640f40cce --- M pywikibot/config2.py M pywikibot/data/mysql.py M pywikibot/pagegenerators.py M scripts/replace.py M scripts/table2wiki.py 5 files changed, 54 insertions(+), 34 deletions(-)
Approvals: Zhuyifei1999: Looks good to me, approved jenkins-bot: Verified
diff --git a/pywikibot/config2.py b/pywikibot/config2.py index 11d2ae4..bd66798 100644 --- a/pywikibot/config2.py +++ b/pywikibot/config2.py @@ -685,7 +685,7 @@ # Setting to connect the database or replica of the database of the wiki. # db_name_format can be used to manipulate the dbName of site. # Example for a pywikibot running on wmflabs: -# db_hostname = 'enwiki.labsdb' +# db_hostname = 'enwiki.analytics.db.svc.eqiad.wmflabs' # db_name_format = '{0}_p' # db_connect_file = user_home_path('replica.my.cnf') db_hostname = 'localhost' @@ -694,7 +694,8 @@ db_name_format = '{0}' db_connect_file = user_home_path('.my.cnf') # local port for mysql server -# ssh -L 4711:enwiki.labsdb:3306 user@tools-login.wmflabs.org +# ssh -L 4711:enwiki.analytics.db.svc.eqiad.wmflabs:3306 \ +# user@login.tools.wmflabs.org db_port = 3306
# ############# SEARCH ENGINE SETTINGS ############## diff --git a/pywikibot/data/mysql.py b/pywikibot/data/mysql.py index c3757ce..e07b180 100644 --- a/pywikibot/data/mysql.py +++ b/pywikibot/data/mysql.py @@ -14,9 +14,14 @@ try: import pymysql as mysqldb except ImportError: - import MySQLdb as mysqldb - pywikibot.warning('PyMySql not found.') - pywikibot.warning('MySQLdb is deprecated. Use PyMySql instead.') + try: + import MySQLdb as mysqldb + except ImportError: + raise ImportError('No supported MySQL library installed. ' + 'Please install PyMySQL.') + else: + pywikibot.warning("PyMySQL not found. It'll fallback " + 'on the deprecated library MySQLdb.') else: mysqldb.install_as_MySQLdb()
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index 180b6a0..5cd45f9 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -234,9 +234,12 @@ Argument can be given as "-withoutinterwiki:n" where n is the total to fetch.
--mysqlquery Takes a Mysql query string like - "SELECT page_namespace, page_title, FROM page - WHERE page_namespace = 0" and works on the resulting pages. +-mysqlquery Takes a MySQL query string like + "SELECT page_namespace, page_title FROM page + WHERE page_namespace = 0" and treats + the resulting pages. See + https://www.mediawiki.org/wiki/Manual:Pywikibot/MySQL + for more details.
-sparql Takes a SPARQL SELECT query string including ?item and works on the resulting pages. diff --git a/scripts/replace.py b/scripts/replace.py index fe56b4c..fa8464c 100755 --- a/scripts/replace.py +++ b/scripts/replace.py @@ -12,9 +12,14 @@
Furthermore, the following command line parameters are supported:
--xml Retrieve information from a local XML dump (pages-articles - or pages-meta-current, see https://dumps.wikimedia.org). - Argument can also be given as "-xml:filename". +-mysqlquery Retrieve information from a local database mirror. + If no query specified, bot searches for pages with + given replacements. + +-xml Retrieve information from a local XML dump + (pages-articles or pages-meta-current, see + https://dumps.wikimedia.org). Argument can also + be given as "-xml:filename".
-regex Make replacements using regular expressions. If this argument isn't given, the bot will make simple text replacements. @@ -890,6 +895,7 @@ # if -xml flag is present xmlFilename = None useSql = False + sql_query = None # will become True when the user presses a ('yes to all') or uses the # -always flag. acceptall = False @@ -933,8 +939,12 @@ xmlFilename = i18n.input('pywikibot-enter-xml-filename') else: xmlFilename = arg[5:] - elif arg == '-sql': + elif arg.startswith(('-sql', '-mysqlquery')): + if arg.startswith('-sql'): + issue_deprecation_warning('The usage of "-sql"', '-mysqlquery', + 1, ArgumentDeprecationWarning) useSql = True + sql_query = arg.partition(':')[2] elif arg.startswith('-excepttitle:'): exceptions['title'].append(arg[13:]) elif arg.startswith('-requiretitle:'): @@ -1155,16 +1165,18 @@ gen = XmlDumpReplacePageGenerator(xmlFilename, xmlStart, replacements, exceptions, site) elif useSql: - whereClause = 'WHERE (%s)' % ' OR '.join( - ["old_text RLIKE '%s'" % prepareRegexForMySQL(old_regexp.pattern) - for (old_regexp, new_text) in replacements]) - if exceptions: - exceptClause = 'AND NOT (%s)' % ' OR '.join( - ["old_text RLIKE '%s'" % prepareRegexForMySQL(exc.pattern) - for exc in exceptions]) - else: - exceptClause = '' - query = u""" + if not sql_query: + whereClause = 'WHERE (%s)' % ' OR '.join( + ["old_text RLIKE '%s'" % prepareRegexForMySQL( + old_regexp.pattern) for (old_regexp, new_text) + in replacements]) + if exceptions: + exceptClause = 'AND NOT (%s)' % ' OR '.join( + ["old_text RLIKE '%s'" % prepareRegexForMySQL(exc.pattern) + for exc in exceptions]) + else: + exceptClause = '' + query = sql_query or """ SELECT page_namespace, page_title FROM page JOIN text ON (page_id = old_id) diff --git a/scripts/table2wiki.py b/scripts/table2wiki.py index 2cbe443..9c6e576 100644 --- a/scripts/table2wiki.py +++ b/scripts/table2wiki.py @@ -7,22 +7,24 @@
¶ms;
--always The bot won't ask for confirmation when putting a page +-always The bot won't ask for confirmation when putting + a page.
-skipwarning Skip processing a page when a warning occurred. Only used when -always is or becomes True.
--quiet Don't show diffs in -always mode +-quiet Don't show diffs in -always mode.
--mysqlquery Retrieve information from a local mirror. - Searches for pages with HTML tables, and tries to convert - them on the live wiki. +-mysqlquery Retrieve information from a local database mirror. + If no query specified, bot searches for pages with + HTML tables, and tries to convert them on the live + wiki.
-xml Retrieve information from a local XML dump (pages_current, see https://dumps.wikimedia.org). Argument can also be given as "-xml:filename". - Searches for pages with HTML tables, and tries to convert - them on the live wiki. + Searches for pages with HTML tables, and tries + to convert them on the live wiki.
Example:
@@ -57,7 +59,7 @@ from pywikibot.bot import (SingleSiteBot, ExistingPageBot, NoRedirectPageBot, suggest_help, input_yn) from pywikibot.exceptions import ArgumentDeprecationWarning -from pywikibot.tools import has_module, issue_deprecation_warning +from pywikibot.tools import issue_deprecation_warning
# This is required for the text that is shown when you run this script # with the parameter -help. @@ -539,10 +541,7 @@ elif option in ['-always', '-quiet', '-skipwarning']: options[option[1:]] = True else: - if option in ['-sql', '-mysqlquery']: - if not (has_module('oursql') or has_module('MySQLdb')): - raise NotImplementedError( - 'Neither "oursql" nor "MySQLdb" library is installed.') + if option in ('-sql', '-mysqlquery'): if option == '-sql': issue_deprecation_warning( 'The usage of "-sql"', '-mysqlquery',