jenkins-bot has submitted this change and it was merged. (
https://gerrit.wikimedia.org/r/272990 )
Change subject: [IMPR] Add documentation for sql parameter in replace.py
......................................................................
[IMPR] Add documentation for sql parameter in replace.py
+ also unify -sql parameters to -mysqlquery between scripts
+ move ImportError from scripts/table2wiki.py to pywikibot/data/mysql.py
+ improve documentation
+ narrow database replica hostname for Toolforge
Bug: T124869
Bug: T182523
Change-Id: I257edc61c6919542227e4b8c47799af640f40cce
---
M pywikibot/config2.py
M pywikibot/data/mysql.py
M pywikibot/pagegenerators.py
M scripts/replace.py
M scripts/table2wiki.py
5 files changed, 54 insertions(+), 34 deletions(-)
Approvals:
Zhuyifei1999: Looks good to me, approved
jenkins-bot: Verified
diff --git a/pywikibot/config2.py b/pywikibot/config2.py
index 11d2ae4..bd66798 100644
--- a/pywikibot/config2.py
+++ b/pywikibot/config2.py
@@ -685,7 +685,7 @@
# Setting to connect the database or replica of the database of the wiki.
# db_name_format can be used to manipulate the dbName of site.
# Example for a pywikibot running on wmflabs:
-# db_hostname = 'enwiki.labsdb'
+# db_hostname = 'enwiki.analytics.db.svc.eqiad.wmflabs'
# db_name_format = '{0}_p'
# db_connect_file = user_home_path('replica.my.cnf')
db_hostname = 'localhost'
@@ -694,7 +694,8 @@
db_name_format = '{0}'
db_connect_file = user_home_path('.my.cnf')
# local port for mysql server
-# ssh -L 4711:enwiki.labsdb:3306 user(a)tools-login.wmflabs.org
+# ssh -L 4711:enwiki.analytics.db.svc.eqiad.wmflabs:3306 \
+# user(a)login.tools.wmflabs.org
db_port = 3306
# ############# SEARCH ENGINE SETTINGS ##############
diff --git a/pywikibot/data/mysql.py b/pywikibot/data/mysql.py
index c3757ce..e07b180 100644
--- a/pywikibot/data/mysql.py
+++ b/pywikibot/data/mysql.py
@@ -14,9 +14,14 @@
try:
import pymysql as mysqldb
except ImportError:
- import MySQLdb as mysqldb
- pywikibot.warning('PyMySql not found.')
- pywikibot.warning('MySQLdb is deprecated. Use PyMySql instead.')
+ try:
+ import MySQLdb as mysqldb
+ except ImportError:
+ raise ImportError('No supported MySQL library installed. '
+ 'Please install PyMySQL.')
+ else:
+ pywikibot.warning("PyMySQL not found. It'll fallback "
+ 'on the deprecated library MySQLdb.')
else:
mysqldb.install_as_MySQLdb()
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py
index 180b6a0..5cd45f9 100644
--- a/pywikibot/pagegenerators.py
+++ b/pywikibot/pagegenerators.py
@@ -234,9 +234,12 @@
Argument can be given as "-withoutinterwiki:n" where
n is the total to fetch.
--mysqlquery Takes a Mysql query string like
- "SELECT page_namespace, page_title, FROM page
- WHERE page_namespace = 0" and works on the resulting pages.
+-mysqlquery Takes a MySQL query string like
+ "SELECT page_namespace, page_title FROM page
+ WHERE page_namespace = 0" and treats
+ the resulting pages. See
+
https://www.mediawiki.org/wiki/Manual:Pywikibot/MySQL
+ for more details.
-sparql Takes a SPARQL SELECT query string including ?item
and works on the resulting pages.
diff --git a/scripts/replace.py b/scripts/replace.py
index fe56b4c..fa8464c 100755
--- a/scripts/replace.py
+++ b/scripts/replace.py
@@ -12,9 +12,14 @@
Furthermore, the following command line parameters are supported:
--xml Retrieve information from a local XML dump (pages-articles
- or pages-meta-current, see
https://dumps.wikimedia.org).
- Argument can also be given as "-xml:filename".
+-mysqlquery Retrieve information from a local database mirror.
+ If no query specified, bot searches for pages with
+ given replacements.
+
+-xml Retrieve information from a local XML dump
+ (pages-articles or pages-meta-current, see
+
https://dumps.wikimedia.org). Argument can also
+ be given as "-xml:filename".
-regex Make replacements using regular expressions. If this argument
isn't given, the bot will make simple text replacements.
@@ -890,6 +895,7 @@
# if -xml flag is present
xmlFilename = None
useSql = False
+ sql_query = None
# will become True when the user presses a ('yes to all') or uses the
# -always flag.
acceptall = False
@@ -933,8 +939,12 @@
xmlFilename = i18n.input('pywikibot-enter-xml-filename')
else:
xmlFilename = arg[5:]
- elif arg == '-sql':
+ elif arg.startswith(('-sql', '-mysqlquery')):
+ if arg.startswith('-sql'):
+ issue_deprecation_warning('The usage of "-sql"',
'-mysqlquery',
+ 1, ArgumentDeprecationWarning)
useSql = True
+ sql_query = arg.partition(':')[2]
elif arg.startswith('-excepttitle:'):
exceptions['title'].append(arg[13:])
elif arg.startswith('-requiretitle:'):
@@ -1155,16 +1165,18 @@
gen = XmlDumpReplacePageGenerator(xmlFilename, xmlStart,
replacements, exceptions, site)
elif useSql:
- whereClause = 'WHERE (%s)' % ' OR '.join(
- ["old_text RLIKE '%s'" %
prepareRegexForMySQL(old_regexp.pattern)
- for (old_regexp, new_text) in replacements])
- if exceptions:
- exceptClause = 'AND NOT (%s)' % ' OR '.join(
- ["old_text RLIKE '%s'" %
prepareRegexForMySQL(exc.pattern)
- for exc in exceptions])
- else:
- exceptClause = ''
- query = u"""
+ if not sql_query:
+ whereClause = 'WHERE (%s)' % ' OR '.join(
+ ["old_text RLIKE '%s'" % prepareRegexForMySQL(
+ old_regexp.pattern) for (old_regexp, new_text)
+ in replacements])
+ if exceptions:
+ exceptClause = 'AND NOT (%s)' % ' OR '.join(
+ ["old_text RLIKE '%s'" %
prepareRegexForMySQL(exc.pattern)
+ for exc in exceptions])
+ else:
+ exceptClause = ''
+ query = sql_query or """
SELECT page_namespace, page_title
FROM page
JOIN text ON (page_id = old_id)
diff --git a/scripts/table2wiki.py b/scripts/table2wiki.py
index 2cbe443..9c6e576 100644
--- a/scripts/table2wiki.py
+++ b/scripts/table2wiki.py
@@ -7,22 +7,24 @@
¶ms;
--always The bot won't ask for confirmation when putting a page
+-always The bot won't ask for confirmation when putting
+ a page.
-skipwarning Skip processing a page when a warning occurred.
Only used when -always is or becomes True.
--quiet Don't show diffs in -always mode
+-quiet Don't show diffs in -always mode.
--mysqlquery Retrieve information from a local mirror.
- Searches for pages with HTML tables, and tries to convert
- them on the live wiki.
+-mysqlquery Retrieve information from a local database mirror.
+ If no query specified, bot searches for pages with
+ HTML tables, and tries to convert them on the live
+ wiki.
-xml Retrieve information from a local XML dump
(pages_current, see
https://dumps.wikimedia.org).
Argument can also be given as "-xml:filename".
- Searches for pages with HTML tables, and tries to convert
- them on the live wiki.
+ Searches for pages with HTML tables, and tries
+ to convert them on the live wiki.
Example:
@@ -57,7 +59,7 @@
from pywikibot.bot import (SingleSiteBot, ExistingPageBot, NoRedirectPageBot,
suggest_help, input_yn)
from pywikibot.exceptions import ArgumentDeprecationWarning
-from pywikibot.tools import has_module, issue_deprecation_warning
+from pywikibot.tools import issue_deprecation_warning
# This is required for the text that is shown when you run this script
# with the parameter -help.
@@ -539,10 +541,7 @@
elif option in ['-always', '-quiet', '-skipwarning']:
options[option[1:]] = True
else:
- if option in ['-sql', '-mysqlquery']:
- if not (has_module('oursql') or has_module('MySQLdb')):
- raise NotImplementedError(
- 'Neither "oursql" nor "MySQLdb" library
is installed.')
+ if option in ('-sql', '-mysqlquery'):
if option == '-sql':
issue_deprecation_warning(
'The usage of "-sql"', '-mysqlquery',
--
To view, visit
https://gerrit.wikimedia.org/r/272990
To unsubscribe, or for help writing mail filters, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-MessageType: merged
Gerrit-Change-Id: I257edc61c6919542227e4b8c47799af640f40cce
Gerrit-Change-Number: 272990
Gerrit-PatchSet: 17
Gerrit-Owner: UltimateSupreme <ultimatesupreme2212(a)gmail.com>
Gerrit-Reviewer: Dvorapa <dvorapa(a)seznam.cz>
Gerrit-Reviewer: Framawiki <framawiki(a)tools.wmflabs.org>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Liuxinyu970226 <541329866(a)qq.com>
Gerrit-Reviewer: Magul <tomasz.magulski(a)gmail.com>
Gerrit-Reviewer: TerraCodes <terracodes(a)tools.wmflabs.org>
Gerrit-Reviewer: Xqt <info(a)gno.de>
Gerrit-Reviewer: Zhuyifei1999 <zhuyifei1999(a)gmail.com>
Gerrit-Reviewer: Zoranzoki21 <zorandori4444(a)gmail.com>
Gerrit-Reviewer: jenkins-bot