jenkins-bot has submitted this change and it was merged.
Change subject: [FEAT] Rewritten generate_user_files and Py3 support
......................................................................
[FEAT] Rewritten generate_user_files and Py3 support
This is a general overhaul of generate_user_files. It uses the pywikibot
package so it can use the config to get the base directory and directly
use inputChoice, input, output and error.
This also uses the Family instances and does not interpret the source
code on it's own. The languages are now sorted by 'languages_by_size'
with any other language added afterwards sorted alphabetically.
It now also requires a non empty username (and doesn't default to
'UnnamedBot'). The default values when adding additional projects are
those of the main. The default language of a project which only has one
(e.g. commons) is that language (and not 'en'). It'll require a
confirmation when the language is not known (and there are languages
known).
All usernames are now by design together. Previously they could
have been separated by other config values in the extended mode.
It also supports now Python 3.
Change-Id: I5d8209d2357a6ebe90a55766a8ff27e953647738
---
M generate_user_files.py
M tox.ini
2 files changed, 202 insertions(+), 204 deletions(-)
Approvals:
John Vandenberg: Looks good to me, approved
jenkins-bot: Verified
diff --git a/generate_user_files.py b/generate_user_files.py
index d38cca0..05df8f0 100644
--- a/generate_user_files.py
+++ b/generate_user_files.py
@@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
-""" Script to create user files (user-config.py, user-fixes.py)
"""
+""" Script to create user files (user-config.py, user-fixes.py).
"""
#
# (C) Pywikibot team, 2010-2014
#
@@ -12,55 +12,14 @@
import sys
import re
import codecs
-import platform
+import math
-single_wiki_families = ['commons', 'wikidata']
+# Disable user-config usage as we are creating it here
+os.environ['PYWIKIBOT2_NO_USER_CONFIG'] = '1'
+import pywikibot
-def get_base_dir():
- """Return the directory in which user-specific information is stored.
-
- This is determined in the following order -
- 1. If the script was called with a -dir: argument, use the directory
- provided in this argument
- 2. If the user has a PYWIKIBOT2_DIR environment variable, use the value
- of it
- 3. Use (and if necessary create) a 'pywikibot' folder under
- 'Application Data' or 'AppData\Roaming' (Windows) or
- '.pywikibot' directory (Unix and similar) under the user's home
- directory.
-
- """
- # copied from config2.py, without the lines that check whether the
- # directory already contains a user-config.py file
- # this code duplication is nasty, should fix
- NAME = u"pywikibot"
- for arg in sys.argv[1:]:
- if arg.startswith("-dir:"):
- base_dir = arg[5:]
- sys.argv.remove(arg)
- break
- else:
- if "PYWIKIBOT2_DIR" in os.environ:
- base_dir = os.environ["PYWIKIBOT2_DIR"]
- else:
- is_windows = sys.platform == 'win32'
- home = os.path.expanduser("~")
- if is_windows:
- _win_version = int(platform.version()[0])
- if _win_version == 5:
- base_dir = os.path.join(home, "Application Data", NAME)
- elif _win_version == 6:
- base_dir = os.path.join(home, "AppData\\Roaming", NAME)
- else:
- base_dir = os.path.join(home, "." + NAME)
- if not os.path.isdir(base_dir):
- os.makedirs(base_dir, mode=0o700)
- if not os.path.isabs(base_dir):
- base_dir = os.path.normpath(os.path.join(os.getcwd(), base_dir))
- return base_dir
-
-base_dir = get_base_dir()
+base_dir = pywikibot.config2.base_dir
console_encoding = sys.stdout.encoding
# the directory in which generate_user_files.py is located
pywikibot_dir = sys.path[0]
@@ -70,6 +29,7 @@
def listchoice(clist, message=None, default=None):
+ """Ask the user to select one entry from a list of
entries."""
if not message:
message = u"Select"
@@ -78,54 +38,52 @@
message += u": "
+ line_template = u"{{0: >{0}}}: {{1}}".format(int(math.log10(len(clist))
+ 1))
for n, i in enumerate(clist):
- print(u"%d: %s" % (n + 1, i))
+ pywikibot.output(line_template.format(n + 1, i))
while True:
- choice = raw_input(message)
+ choice = pywikibot.input(message)
if choice == '' and default:
return default
try:
- choice = int(choice)
+ choice = int(choice) - 1
except ValueError:
- # User typed choice name
- if choice not in clist:
- print("Invalid response")
- continue
- else:
- return choice
+ try:
+ choice = clist.index(choice)
+ except IndexError:
+ choice = -1
# User typed choice number
- if 1 <= choice <= len(clist):
- return clist[choice - 1]
+ if 0 <= choice < len(clist):
+ return clist[choice]
else:
- print("Invalid response")
+ pywikibot.error("Invalid response")
def change_base_dir():
"""Create a new user directory."""
- global base_dir
while True:
- new_base = raw_input("New user directory? ")
+ new_base = pywikibot.input("New user directory? ")
new_base = os.path.abspath(new_base)
if os.path.exists(new_base):
if os.path.isfile(new_base):
- print("ERROR: there is an existing file with that name.")
+ pywikibot.error("there is an existing file with that name.")
continue
# make sure user can read and write this directory
if not os.access(new_base, os.R_OK | os.W_OK):
- print("ERROR: directory access restricted")
+ pywikibot.error("directory access restricted")
continue
- print("OK: using existing directory")
+ pywikibot.output("Using existing directory")
break
else:
try:
os.mkdir(new_base, 0o700)
except Exception:
- print("ERROR: directory creation failed")
+ pywikibot.error("ERROR: directory creation failed")
continue
- print("OK: Created new directory.")
+ pywikibot.output("Created new directory.")
break
from textwrap import wrap
@@ -136,104 +94,101 @@
your operating system. See your operating system documentation for how to
set environment variables.""" % locals(), width=76)
for line in msg:
- print(line)
- ok = raw_input("Is this OK? ([y]es, [N]o) ")
- if ok in ["Y", "y"]:
- base_dir = new_base
- return True
- print("Aborting changes.")
+ pywikibot.output(line)
+ if pywikibot.inputChoice("Is this OK?", ["yes", "no"],
["y", "n"], "n") == "y":
+ return new_base
+ pywikibot.output("Aborting changes.")
return False
def file_exists(filename):
+ """Return whether the file exists and print a message if it
exists."""
if os.path.exists(filename):
- print(u"'%s' already exists." % filename)
+ pywikibot.output(u"'%s' already exists." % filename)
return True
return False
-def get_site_and_lang():
- known_families = re.findall(
- r'(.+)_family.py\b',
- '\n'.join(os.listdir(os.path.join(
- pywikibot_dir,
- "pywikibot",
- "families")))
- )
- known_families = sorted(known_families)
- fam = listchoice(known_families,
- u"Select family of sites we are working on, "
- u"just enter the number not name",
- default=u'wikipedia')
- if fam not in single_wiki_families:
- codesds = codecs.open(os.path.join(pywikibot_dir,
- u"pywikibot",
- u"families",
- u"%s_family.py" % fam),
- "r", "utf-8").read()
- rre = re.compile(u"self\.languages\_by\_size *\= *(.+?)\]",
- re.DOTALL)
- known_langs = []
- if not rre.findall(codesds):
- rre = re.compile(u"self\.langs *\= *(.+?)\}", re.DOTALL)
- if rre.findall(codesds):
- import ast
- known_langs = ast.literal_eval(
- rre.findall(codesds)[0] + u"}").keys()
- else:
- known_langs = eval(rre.findall(codesds)[0] + u"]")
- print("This is the list of known language(s):")
- print(u" ".join(sorted(known_langs)))
- mylang = raw_input("The language code of the site we're working on
"
- "(default: 'en'): ") or 'en'
+def get_site_and_lang(default_family='wikipedia', default_lang='en',
+ default_username=None):
+ """
+ Ask the user for the family, language and username.
+
+ @param default_family: The default family which should be chosen.
+ @type default_family: None or str
+ @param default_lang: The default language which should be chosen, if the
+ family supports this language.
+ @type default_lang: None or str
+ @param default_username: The default username which should be chosen.
+ @type default_username: None or str
+ @return: The family, language and username
+ @rtype: tuple of three str
+ """
+ known_families = sorted(pywikibot.config2.family_files.keys())
+ if default_family not in known_families:
+ default_family = None
+ fam = listchoice(known_families,
+ u"Select family of sites we are working on, "
+ u"just enter the number or name",
+ default=default_family)
+ fam = pywikibot.family.Family.load(fam)
+ if hasattr(fam, "langs"):
+ if hasattr(fam, "languages_by_size"):
+ key = {lang: i for i, lang in enumerate(fam.languages_by_size)}.get
+ by_size = sorted(
+ set(fam.langs.keys()).intersection(fam.languages_by_size),
+ key=key)
else:
- mylang = fam
+ by_size = []
+ known_langs = by_size + sorted(
+ set(fam.langs.keys()).difference(by_size))
+ else:
+ known_langs = []
- username = raw_input(u"Username (%s %s): "
- % (mylang, fam))
- username = unicode(username, console_encoding)
- return fam, mylang, username
+ if len(known_langs) == 0:
+ pywikibot.output('There were no known languages found in
{0}.'.format(fam.name))
+ default_lang = None
+ elif len(known_langs) == 1:
+ pywikibot.output('The only known language: {0}'.format(known_langs[0]))
+ default_lang = known_langs[0]
+ else:
+ pywikibot.output("This is the list of known languages:")
+ pywikibot.output(u", ".join(known_langs))
+ if default_lang not in known_langs:
+ if default_lang != 'en' and 'en' in known_langs:
+ default_lang = 'en'
+ else:
+ default_lang = None
+ message = "The language code of the site we're working on"
+ if default_lang:
+ message += " (default: '{0}')".format(default_lang)
+ message += ":"
+ mylang = None
+ while not mylang:
+ mylang = pywikibot.input(message) or default_lang
+ if known_langs and mylang and mylang not in known_langs:
+ if pywikibot.inputChoice("The language code {0} is not in the "
+ "list of known languages. Do you want to
"
+ "continue?".format(mylang),
+ ["yes", "no"], ["y",
"n"], "n") == "n":
+ mylang = None
+ username = None
+ message = u"Username on {0}:{1}".format(mylang, fam.name)
+ if default_username:
+ message += " (default: '{0}')".format(default_username)
+ message += ":"
+ while not username:
+ username = pywikibot.input(message) or default_username
+ if not username:
+ pywikibot.error('The username may not be empty.')
+ if sys.version_info == 2:
+ username = username.decode(console_encoding)
+ # Escape ''s
+ username = username.replace("'", "\\'")
+ return fam.name, mylang, username
-def create_user_config():
- _fnc = os.path.join(base_dir, "user-config.py")
- if not file_exists(_fnc):
- fam, mylang, mainusername = get_site_and_lang()
- mainusername = mainusername or "UnnamedBot"
-
- while True:
- choice = raw_input(
- "Which variant of user_config.py:\n"
- "[S]mall or [E]xtended (with further information)? ").upper()
- if choice in "SE":
- break
-
- # determine what directory this script (generate_user_files.py) lives in
- install = os.path.dirname(os.path.abspath(__file__))
- # config2.py will be in the pywikibot/ directory
- f = codecs.open(os.path.join(install, "pywikibot",
"config2.py"),
- "r", "utf-8")
- cpy = f.read()
- f.close()
-
- res = re.findall("^(############## (?:"
- "LOGFILE|"
- "INTERWIKI|"
- "SOLVE_DISAMBIGUATION|"
- "IMAGE RELATED|"
- "TABLE CONVERSION BOT|"
- "WEBLINK CHECKER|"
- "DATABASE|"
- "SEARCH ENGINE|"
- "COPYRIGHT|"
- "FURTHER"
- ") SETTINGS .*?)^(?=#####|# =====)",
- cpy, re.MULTILINE | re.DOTALL)
- config_text = '\n'.join(res)
-
- f = codecs.open(_fnc, "w", "utf-8")
- if choice == 'E':
- f.write(u"""# -*- coding: utf-8 -*-
+EXTENDED_CONFIG = u"""# -*- coding: utf-8 -*-
# This is an automatically generated file. You can find more configuration
# parameters in 'config.py' file.
@@ -266,38 +221,78 @@
# If you use either of these functions to define the family to work on by
# default (the ‘family’ variable below), you must place the function call
# before the definition of the ‘family’ variable.
-family = '%s'
+family = '{main_family}'
# The language code of the site we're working on.
-mylang = '%s'
+mylang = '{main_lang}'
# The dictionary usernames should contain a username for each site where you
# have a bot account. If you have a unique username for all languages of a
# family , you can use '*'
-usernames['%s']['%s'] = u'%s'
+{usernames}
-%s""" % (fam, mylang, fam, mylang, mainusername, config_text))
+{config_text}"""
+
+SMALL_CONFIG = (u"# -*- coding: utf-8 -*-\n"
+ u"family = '{main_family}'\n"
+ u"mylang = '{main_lang}'\n"
+ u"{usernames}\n")
+
+
+def create_user_config():
+ """Create a user-config.py in base_dir."""
+ _fnc = os.path.join(base_dir, "user-config.py")
+ if not file_exists(_fnc):
+ main_family, main_lang, main_username = get_site_and_lang()
+
+ usernames = [(main_family, main_lang, main_username)]
+ while pywikibot.inputChoice("Do you want to add any other projects?",
+ ["yes", "no"], ["y",
"n"], "n") == "y":
+ usernames += [get_site_and_lang(main_family, main_lang,
+ main_username)]
+
+ usernames = '\n'.join(
+ u"usernames['{0}']['{1}'] =
u'{2}'".format(*username)
+ for username in usernames)
+
+ extended = pywikibot.inputChoice(
+ "Which variant of user_config.py?",
+ ["small", "extended (with further information)"],
["s", "e"]) == "e"
+
+ if extended:
+ # config2.py will be in the pywikibot/ directory relative to this
+ # script (generate_user_files)
+ install = os.path.dirname(os.path.abspath(__file__))
+ with codecs.open(os.path.join(install, "pywikibot",
"config2.py"),
+ "r", "utf-8") as config_f:
+ config = config_f.read()
+
+ res = re.findall("^(############## (?:"
+ "LOGFILE|"
+ "INTERWIKI|"
+ "SOLVE_DISAMBIGUATION|"
+ "IMAGE RELATED|"
+ "TABLE CONVERSION BOT|"
+ "WEBLINK CHECKER|"
+ "DATABASE|"
+ "SEARCH ENGINE|"
+ "COPYRIGHT|"
+ "FURTHER"
+ ") SETTINGS .*?)^(?=#####|# =====)",
+ config, re.MULTILINE | re.DOTALL)
+ config_text = '\n'.join(res)
+ config_content = EXTENDED_CONFIG
else:
- f.write(u"""# -*- coding: utf-8 -*-
-family = '%s'
-mylang = '%s'
-usernames['%s']['%s'] = u'%s'
-""" % (fam, mylang, fam, mylang, mainusername))
- while(raw_input(
- "Do you want to add any other projects? (y/N)").upper() ==
"Y"):
- fam, mylang, username = get_site_and_lang()
- username = username or mainusername
- # Escape ''s
- username = username.replace("'", "\\'")
- f.write(u"usernames['%(fam)s']['%(mylang)s'] =
u'%(username)s'\n"
- % locals())
+ config_content = SMALL_CONFIG
- f.close()
- print(u"'%s' written." % _fnc)
+ with codecs.open(_fnc, "w", "utf-8") as f:
+ f.write(config_content.format(**locals()))
+ pywikibot.output(u"'%s' written." % _fnc)
def create_user_fixes():
+ """Create a basic user-fixes.py in base_dir."""
_fnf = os.path.join(base_dir, "user-fixes.py")
if not file_exists(_fnf):
with codecs.open(_fnf, "w", "utf-8") as f:
@@ -318,59 +313,61 @@
}
""")
- print(u"'%s' written." % _fnf)
+ pywikibot.output(u"'%s' written." % _fnf)
if __name__ == "__main__":
while True:
- print(u'\nYour default user directory is "%s"' % base_dir)
- ok = raw_input("How to proceed? ([K]eep [c]hange) ").upper().strip()
- if (not ok) or "KEEP".startswith(ok):
- break
- if "CHANGE".startswith(ok):
- if change_base_dir():
+ pywikibot.output(u'\nYour default user directory is "%s"' %
base_dir)
+ if pywikibot.inputChoice("How to proceed?", ["keep",
"change"],
+ ["k", "c"], "k") ==
"c":
+ new_base = change_base_dir()
+ if new_base:
+ base_dir = new_base
break
+ else:
+ break
+
+ copied_config = False
+ copied_fixes = False
while True:
if os.path.exists(os.path.join(base_dir, "user-config.py")):
break
- do_copy = raw_input("Do you want to copy user files from an existing "
- "Pywikibot installation? ").upper().strip()
- if do_copy and "YES".startswith(do_copy):
- oldpath = raw_input("Path to existing wikipedia.py? ")
+ if pywikibot.inputChoice(
+ "Do you want to copy user files from an existing Pywikibot "
+ "installation?", ["yes", "no"],
["y", "n"]) == "y":
+ oldpath = pywikibot.input("Path to existing wikipedia.py?")
if not os.path.exists(oldpath):
- print("ERROR: Not a valid path")
+ pywikibot.error("Not a valid path")
continue
if os.path.isfile(oldpath):
# User probably typed /wikipedia.py at the end, so strip it
oldpath = os.path.dirname(oldpath)
if not os.path.isfile(os.path.join(oldpath, "user-config.py")):
- print("ERROR: no user_config.py found in that directory")
+ pywikibot.error("No user_config.py found in that directory")
continue
- newf = file(os.path.join(base_dir, "user-config.py"),
"wb")
- oldf = file(os.path.join(oldpath, "user-config.py"),
"rb")
- newf.write(oldf.read())
- newf.close()
- oldf.close()
+ import shutil
+ shutil.copyfile(os.path.join(oldpath, "user-config.py"),
+ os.path.join(base_dir, "user-config.py"))
+ copied_config = True
if os.path.isfile(os.path.join(oldpath, "user-fixes.py")):
- newfix = file(os.path.join(base_dir, "user-fixes.py"),
"wb")
- oldfix = file(os.path.join(oldpath, "user-fixes.py"),
"rb")
- newfix.write(oldfix.read())
- newfix.close()
- oldfix.close()
+ shutil.copyfile(os.path.join(oldpath, "user-fixes.py"),
+ os.path.join(base_dir, "user-fixes.py"))
+ copied_fixes = True
- elif do_copy and "NO".startswith(do_copy):
+ else:
break
if not os.path.isfile(os.path.join(base_dir, "user-config.py")):
- a = raw_input("Create user-config.py file? Required for running bots "
- "([y]es, [N]o) ")
- if a[:1] in ["Y", "y"]:
+ if pywikibot.inputChoice("Create user-config.py file? Required for "
+ "running bots.",
+ ["yes", "no"], ["y",
"n"], "n") == "y":
create_user_config()
- else:
- print("NOTE: user-config.py already exists in the directory")
+ elif not copied_config:
+ pywikibot.output("user-config.py already exists in the directory")
if not os.path.isfile(os.path.join(base_dir, "user-fixes.py")):
- a = raw_input("Create user-fixes.py file? Optional and for advanced "
- "users ([y]es, [N]o) ")
- if a[:1] in ["Y", "y"]:
+ if pywikibot.inputChoice("Create user-fixes.py file? Optional and for
"
+ "advanced users.",
+ ["yes", "no"], ["y",
"n"], "n") == "y":
create_user_fixes()
- else:
- print("NOTE: user-fixes.py already exists in the directory")
+ elif not copied_fixes:
+ pywikibot.output("user-fixes.py already exists in the directory")
diff --git a/tox.ini b/tox.ini
index fd26d1c..ed4e709 100644
--- a/tox.ini
+++ b/tox.ini
@@ -27,6 +27,7 @@
# Add any new file which is pep257 compatible
commands = flake8 --jobs=1 --ignore=D102,D103,E122,E127,E241,E265 \
./setup.py \
+ ./generate_user_files.py \
./pywikibot/__init__.py \
./pywikibot/bot.py \
./pywikibot/botirc.py \
--
To view, visit
https://gerrit.wikimedia.org/r/164299
To unsubscribe, visit
https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: I5d8209d2357a6ebe90a55766a8ff27e953647738
Gerrit-PatchSet: 7
Gerrit-Project: pywikibot/core
Gerrit-Branch: master
Gerrit-Owner: XZise <CommodoreFabianus(a)gmx.de>
Gerrit-Reviewer: John Vandenberg <jayvdb(a)gmail.com>
Gerrit-Reviewer: Ladsgroup <ladsgroup(a)gmail.com>
Gerrit-Reviewer: Merlijn van Deen <valhallasw(a)arctus.nl>
Gerrit-Reviewer: jenkins-bot <>