jenkins-bot has submitted this change and it was merged. ( https://gerrit.wikimedia.org/r/424992 )

Change subject: Create a python installer script for appveyor builds ......................................................................

Create a python installer script for appveyor builds

Currently the installer is not able to install Python 3.5.0+ or 2.7.7-.

Bug: T191188 Change-Id: I19405b7fc5219940f998b6c316c8cafc597ec37e --- M .appveyor.yml A scripts/maintenance/appveyor_python_setup.py 2 files changed, 124 insertions(+), 26 deletions(-)

Approvals: Xqt: Looks good to me, approved jenkins-bot: Verified

diff --git a/.appveyor.yml b/.appveyor.yml index 2755449..6d84aa7 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -2,32 +2,25 @@ environment:

global: - APPVEYOR_PYTHON_URL: "https://raw.githubusercontent.com/ogrisel/python-appveyor-demo/master/appvey..." - - # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the - # /E:ON and /V:ON options are not enabled in the batch script intepreter - # See: http://stackoverflow.com/a/13751649/163740 - CMD_IN_ENV: "cmd /E:ON /V:ON /C .\appveyor\run_with_env.cmd"

PYWIKIBOT2_DIR: "%appdata%\Pywikibot" PYWIKIBOT2_USER_CONFIG: "%appdata%\Pywikibot\user-config.py" - PYSETUP_TEST_EXTRAS: "1"

matrix:

# Test the lowest supported release of each major Python version.

- - PYTHON: "C:\Python272" - PYTHON_VERSION: "2.7.2" + - PYTHON: "C:\Python277" + PYTHON_VERSION: "2.7.7" PYTHON_ARCH: "32"

- PYTHON: "C:\Python340" PYTHON_VERSION: "3.4.0" PYTHON_ARCH: "32"

- - PYTHON: "C:\Python272-x64" - PYTHON_VERSION: "2.7.2" + - PYTHON: "C:\Python277-x64" + PYTHON_VERSION: "2.7.7" PYTHON_ARCH: "64"

- PYTHON: "C:\Python340-x64" @@ -73,19 +66,9 @@

install: - git submodule update --init - # Download the Appveyor Python build accessories into subdirectory .\appveyor - - mkdir appveyor - - ps: $wc = new-object net.webclient - - ps: $run = $wc.DownloadString($env:APPVEYOR_PYTHON_URL + 'run_with_env.cmd') - - ps: $run | Out-File -Encoding ascii -FilePath appveyor\run_with_env.cmd - - # This is needed for Python versions not installed on Appveyor build machines - - ps: if (-not(Test-Path($env:PYTHON))) { iex $wc.DownloadString($env:APPVEYOR_PYTHON_URL + 'install.ps1') } - - pip install virtualenv - - virtualenv env - - env\Scripts\activate.bat - - pip install -r dev-requirements.txt - - pip install -r requests-requirements.txt + - python scripts\maintenance\appveyor_python_setup.py + - pip install -r dev-requirements.txt -r requests-requirements.txt + - where python pip

build: off

@@ -94,11 +77,11 @@ - set PYTHONIOENCODING=utf8

- "mkdir %PYWIKIBOT2_DIR%" - - "python -Werror::UserWarning -m generate_user_files -dir:%PYWIKIBOT2_DIR% -family:wikipedia -lang:en -v -debug" + - "%PYTHON%/python.exe -Werror::UserWarning -m generate_user_files -dir:%PYWIKIBOT2_DIR% -family:wikipedia -lang:en -v -debug" - ps: "[IO.File]::AppendAllText($env:PYWIKIBOT2_USER_CONFIG, 'max_retries = 2; maximum_GET_length = 5000; transliteration_target = None;')"

- set PYSETUP_TEST_NO_UI=1 - - "%CMD_IN_ENV% coverage run setup.py test" + - coverage run setup.py test

on_failure: - codecov diff --git a/scripts/maintenance/appveyor_python_setup.py b/scripts/maintenance/appveyor_python_setup.py new file mode 100644 index 0000000..f5eaaac --- /dev/null +++ b/scripts/maintenance/appveyor_python_setup.py @@ -0,0 +1,115 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +"""Ensure the right version of Python and pip are installed on AppVeyor.""" +from __future__ import ( + absolute_import, division, print_function, unicode_literals) + +from io import open +from os import getenv, listdir +from os.path import exists +from subprocess import CalledProcessError, check_call +try: + from urllib.request import urlretrieve +except ImportError: # Python 2 + from urllib import urlretrieve + + +def get_installer_info(version, arch64): + """Return the download url and filename for given version and arch.""" + # See https://www.python.org/downloads/windows/ + msi_installer = version < '3.5.0' + filename = 'python-{version}{amd64}{extension}'.format( + amd64=(('.' if msi_installer else '-') + + 'amd64' if arch64 else ''), + extension=('.msi' if msi_installer else '.exe'), + **locals()) + url = 'https://www.python.org/ftp/python/%7Bversion%7D/%7Bfilename%7D%27.format( + **locals()) + return filename, url + + +def download_python(version, arch64): + """Download Python installer and return its filename.""" + filename, download_url = get_installer_info(version, arch64) + print('Downloading Python installer from', download_url) + urlretrieve(download_url, filename) + return filename + + +def print_python_install_log(version): + """Print the log for Python installation.""" + # appveyor's %temp% points to "C:\Users\appveyor\AppData\Local\Temp\1" + # but python log files are stored in "C:\Users\appveyor\AppData\Local\Temp + temp_dir = 'C:/Users/appveyor/AppData/Local/Temp/' + for file in listdir(temp_dir): + if file[-4:] == '.log' and file.startswith('Python ' + version): + with open(temp_dir + file, encoding='utf-8') as log: + print(file + ':\n' + log.read()) + break + else: + print('There was no Python log file.') + + +def install_python(installer, python_dir, python_ver): + """Install Python using specified installer file.""" + common_args = [ + installer, '/quiet', 'TargetDir=' + python_dir, 'AssociateFiles=0', + 'Shortcuts=0', 'Include_doc=0', 'Include_launcher=0', + 'InstallLauncherAllUsers=0', 'Include_tcltk=0', 'Include_test=0'] + try: + if installer[-4:] == '.msi': + check_call(['msiexec', '/norestart', '/i'] + common_args) + else: # executable installer + # uninstall first, otherwise the install won't do anything + check_call(common_args) + except CalledProcessError: + print_python_install_log(python_ver) + raise SystemExit('install_python failed') + + +def download_packages(packages, python_dir): + """Download packages and return their paths.""" + preinstalled_py_dir = python_dir[:11] + python_dir[12:] \ + if len(python_dir) in (16, 12) else python_dir + preinstalled_pip = preinstalled_py_dir + '/scripts/pip.exe' + # It is not possible to use --python-version due to cryptography + # requirements on py2.7 which leads to + # https://stackoverflow.com/questions/46287077/ + check_call([ + preinstalled_pip, 'download', '--dest', '.pip_downloads', + '--disable-pip-version-check'] + packages) + downloads = listdir('.pip_downloads') + assert downloads, 'pip did not download anything' + return ['.pip_downloads/' + fn for fn in downloads] + + +def install_packages(python_dir, python_ver): + """Install/upgrade pip, setuptools, and other packages if required.""" + python = python_dir + '/python.exe' + packages = ['pip', 'setuptools'] + pip_installer = [python, '-m', 'pip', 'install', '-U'] + if python_ver < '2.7.9': + check_call([python, 'ez_setup.py']) # bootstrap setuptools + urlretrieve('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py') + pip_installer = [python, '-m', 'get-pip', '-U'] + packages.remove('setuptools') # can't upgrade bootstrapped setuptools + packages.extend(('wheel', 'pip')) + packages = download_packages(packages, python_dir) + check_call(pip_installer + packages) + + +def main(): + python_ver = getenv('PYTHON_VERSION') + python_dir = getenv('PYTHON') + if not python_ver[-2:] == '.x': # .x is for pre-installed Python versions + arch64 = getenv('PYTHON_ARCH') == '64' + filename = download_python(python_ver, arch64) + install_python(filename, python_dir, python_ver) + if not exists(python_dir + r'\python.exe'): + print_python_install_log(python_ver) + raise SystemExit(python_dir + r'\python.exe not found') + install_packages(python_dir, python_ver) + + +if __name__ == '__main__': + main()