jenkins-bot merged this change.
Create a python installer script for appveyor builds
Currently the installer is not able to install Python 3.5.0+ or 2.7.7-.
Bug: T191188
Change-Id: I19405b7fc5219940f998b6c316c8cafc597ec37e
---
M .appveyor.yml
A scripts/maintenance/appveyor_python_setup.py
2 files changed, 124 insertions(+), 26 deletions(-)
diff --git a/.appveyor.yml b/.appveyor.yml
index 2755449..6d84aa7 100644
--- a/.appveyor.yml
+++ b/.appveyor.yml
@@ -2,32 +2,25 @@
environment:
global:
- APPVEYOR_PYTHON_URL: "https://raw.githubusercontent.com/ogrisel/python-appveyor-demo/master/appveyor/"
-
- # SDK v7.0 MSVC Express 2008's SetEnv.cmd script will fail if the
- # /E:ON and /V:ON options are not enabled in the batch script intepreter
- # See: http://stackoverflow.com/a/13751649/163740
- CMD_IN_ENV: "cmd /E:ON /V:ON /C .\\appveyor\\run_with_env.cmd"
PYWIKIBOT2_DIR: "%appdata%\\Pywikibot"
PYWIKIBOT2_USER_CONFIG: "%appdata%\\Pywikibot\\user-config.py"
-
PYSETUP_TEST_EXTRAS: "1"
matrix:
# Test the lowest supported release of each major Python version.
- - PYTHON: "C:\\Python272"
- PYTHON_VERSION: "2.7.2"
+ - PYTHON: "C:\\Python277"
+ PYTHON_VERSION: "2.7.7"
PYTHON_ARCH: "32"
- PYTHON: "C:\\Python340"
PYTHON_VERSION: "3.4.0"
PYTHON_ARCH: "32"
- - PYTHON: "C:\\Python272-x64"
- PYTHON_VERSION: "2.7.2"
+ - PYTHON: "C:\\Python277-x64"
+ PYTHON_VERSION: "2.7.7"
PYTHON_ARCH: "64"
- PYTHON: "C:\\Python340-x64"
@@ -73,19 +66,9 @@
install:
- git submodule update --init
- # Download the Appveyor Python build accessories into subdirectory .\appveyor
- - mkdir appveyor
- - ps: $wc = new-object net.webclient
- - ps: $run = $wc.DownloadString($env:APPVEYOR_PYTHON_URL + 'run_with_env.cmd')
- - ps: $run | Out-File -Encoding ascii -FilePath appveyor\run_with_env.cmd
-
- # This is needed for Python versions not installed on Appveyor build machines
- - ps: if (-not(Test-Path($env:PYTHON))) { iex $wc.DownloadString($env:APPVEYOR_PYTHON_URL + 'install.ps1') }
- - pip install virtualenv
- - virtualenv env
- - env\Scripts\activate.bat
- - pip install -r dev-requirements.txt
- - pip install -r requests-requirements.txt
+ - python scripts\maintenance\appveyor_python_setup.py
+ - pip install -r dev-requirements.txt -r requests-requirements.txt
+ - where python pip
build: off
@@ -94,11 +77,11 @@
- set PYTHONIOENCODING=utf8
- "mkdir %PYWIKIBOT2_DIR%"
- - "python -Werror::UserWarning -m generate_user_files -dir:%PYWIKIBOT2_DIR% -family:wikipedia -lang:en -v -debug"
+ - "%PYTHON%/python.exe -Werror::UserWarning -m generate_user_files -dir:%PYWIKIBOT2_DIR% -family:wikipedia -lang:en -v -debug"
- ps: "[IO.File]::AppendAllText($env:PYWIKIBOT2_USER_CONFIG, 'max_retries = 2; maximum_GET_length = 5000; transliteration_target = None;')"
- set PYSETUP_TEST_NO_UI=1
- - "%CMD_IN_ENV% coverage run setup.py test"
+ - coverage run setup.py test
on_failure:
- codecov
diff --git a/scripts/maintenance/appveyor_python_setup.py b/scripts/maintenance/appveyor_python_setup.py
new file mode 100644
index 0000000..f5eaaac
--- /dev/null
+++ b/scripts/maintenance/appveyor_python_setup.py
@@ -0,0 +1,115 @@
+#!/usr/bin/python
+# -*- coding: utf-8 -*-
+"""Ensure the right version of Python and pip are installed on AppVeyor."""
+from __future__ import (
+ absolute_import, division, print_function, unicode_literals)
+
+from io import open
+from os import getenv, listdir
+from os.path import exists
+from subprocess import CalledProcessError, check_call
+try:
+ from urllib.request import urlretrieve
+except ImportError: # Python 2
+ from urllib import urlretrieve
+
+
+def get_installer_info(version, arch64):
+ """Return the download url and filename for given version and arch."""
+ # See https://www.python.org/downloads/windows/
+ msi_installer = version < '3.5.0'
+ filename = 'python-{version}{amd64}{extension}'.format(
+ amd64=(('.' if msi_installer else '-')
+ + 'amd64' if arch64 else ''),
+ extension=('.msi' if msi_installer else '.exe'),
+ **locals())
+ url = 'https://www.python.org/ftp/python/{version}/{filename}'.format(
+ **locals())
+ return filename, url
+
+
+def download_python(version, arch64):
+ """Download Python installer and return its filename."""
+ filename, download_url = get_installer_info(version, arch64)
+ print('Downloading Python installer from', download_url)
+ urlretrieve(download_url, filename)
+ return filename
+
+
+def print_python_install_log(version):
+ """Print the log for Python installation."""
+ # appveyor's %temp% points to "C:\Users\appveyor\AppData\Local\Temp\1"
+ # but python log files are stored in "C:\Users\appveyor\AppData\Local\Temp
+ temp_dir = 'C:/Users/appveyor/AppData/Local/Temp/'
+ for file in listdir(temp_dir):
+ if file[-4:] == '.log' and file.startswith('Python ' + version):
+ with open(temp_dir + file, encoding='utf-8') as log:
+ print(file + ':\n' + log.read())
+ break
+ else:
+ print('There was no Python log file.')
+
+
+def install_python(installer, python_dir, python_ver):
+ """Install Python using specified installer file."""
+ common_args = [
+ installer, '/quiet', 'TargetDir=' + python_dir, 'AssociateFiles=0',
+ 'Shortcuts=0', 'Include_doc=0', 'Include_launcher=0',
+ 'InstallLauncherAllUsers=0', 'Include_tcltk=0', 'Include_test=0']
+ try:
+ if installer[-4:] == '.msi':
+ check_call(['msiexec', '/norestart', '/i'] + common_args)
+ else: # executable installer
+ # uninstall first, otherwise the install won't do anything
+ check_call(common_args)
+ except CalledProcessError:
+ print_python_install_log(python_ver)
+ raise SystemExit('install_python failed')
+
+
+def download_packages(packages, python_dir):
+ """Download packages and return their paths."""
+ preinstalled_py_dir = python_dir[:11] + python_dir[12:] \
+ if len(python_dir) in (16, 12) else python_dir
+ preinstalled_pip = preinstalled_py_dir + '/scripts/pip.exe'
+ # It is not possible to use --python-version due to cryptography
+ # requirements on py2.7 which leads to
+ # https://stackoverflow.com/questions/46287077/
+ check_call([
+ preinstalled_pip, 'download', '--dest', '.pip_downloads',
+ '--disable-pip-version-check'] + packages)
+ downloads = listdir('.pip_downloads')
+ assert downloads, 'pip did not download anything'
+ return ['.pip_downloads/' + fn for fn in downloads]
+
+
+def install_packages(python_dir, python_ver):
+ """Install/upgrade pip, setuptools, and other packages if required."""
+ python = python_dir + '/python.exe'
+ packages = ['pip', 'setuptools']
+ pip_installer = [python, '-m', 'pip', 'install', '-U']
+ if python_ver < '2.7.9':
+ check_call([python, 'ez_setup.py']) # bootstrap setuptools
+ urlretrieve('https://bootstrap.pypa.io/get-pip.py', 'get-pip.py')
+ pip_installer = [python, '-m', 'get-pip', '-U']
+ packages.remove('setuptools') # can't upgrade bootstrapped setuptools
+ packages.extend(('wheel', 'pip'))
+ packages = download_packages(packages, python_dir)
+ check_call(pip_installer + packages)
+
+
+def main():
+ python_ver = getenv('PYTHON_VERSION')
+ python_dir = getenv('PYTHON')
+ if not python_ver[-2:] == '.x': # .x is for pre-installed Python versions
+ arch64 = getenv('PYTHON_ARCH') == '64'
+ filename = download_python(python_ver, arch64)
+ install_python(filename, python_dir, python_ver)
+ if not exists(python_dir + r'\python.exe'):
+ print_python_install_log(python_ver)
+ raise SystemExit(python_dir + r'\python.exe not found')
+ install_packages(python_dir, python_ver)
+
+
+if __name__ == '__main__':
+ main()
To view, visit change 424992. To unsubscribe, visit settings.