Merlijn van Deen has submitted this change and it was merged.
Change subject: Clean up MySQLPageGenerator ......................................................................
Clean up MySQLPageGenerator
Since we get the database name directly from the site, it probably doesn't match up with the replicated copy that is being used (Toolserver or WMF labs).
Also try using oursql if available, it generally has better unicode support.
Change-Id: Ied3e7b3441ce0b6af500ae9ac6c75c1ba92e13f1 --- M pywikibot/pagegenerators.py 1 file changed, 31 insertions(+), 5 deletions(-)
Approvals: Xqt: Looks good to me, but someone else must approve Merlijn van Deen: Looks good to me, approved
diff --git a/pywikibot/pagegenerators.py b/pywikibot/pagegenerators.py index f25f827..8318048 100644 --- a/pywikibot/pagegenerators.py +++ b/pywikibot/pagegenerators.py @@ -1056,13 +1056,38 @@ if page.site == self.site: yield page
-def MySQLPageGenerator(query, site = None): - import MySQLdb as mysqldb + +def MySQLPageGenerator(query, site=None): + """ + Requires oursql http://pythonhosted.org/oursql/ or + MySQLdb https://sourceforge.net/projects/mysql-python/ + Yields a list of pages based on a MySQL query. Each query + should provide the page namespace and page title. An example + query that yields all ns0 pages might look like: + SELECT + page_namespace, + page_title, + FROM page + WHERE page_namespace = 0; + @param query: MySQL query to execute + @param site: Site object or raw database name + @type site: pywikibot.Site|str + @return: iterator of pywikibot.Page + """ + try: + import oursql as mysqldb + except ImportError: + import MySQLdb as mysqldb if site is None: site = pywikibot.Site() - conn = mysqldb.connect(config.db_hostname, db = site.dbName(), - user = config.db_username, - passwd = config.db_password) + if isinstance(site, pywikibot.site.Site): + # We want to let people to set a custom dbname + # since the master dbname might not be exactly + # equal to the name on the replicated site + site = site.dbName() + conn = mysqldb.connect(config.db_hostname, db=site, + user=config.db_username, + passwd=config.db_password) cursor = conn.cursor() pywikibot.output(u'Executing query:\n%s' % query) query = query.encode(site.encoding()) @@ -1085,6 +1110,7 @@ page = pywikibot.Page(site, pageTitle) yield page
+ def YearPageGenerator(start = 1, end = 2050, site = None): if site is None: site = pywikibot.Site()