# coding: utf-8
from __future__ import unicode_literals, division, absolute_import, print_function
import os
import subprocess
import sys
import shutil
import re
import json
import tarfile
import zipfile
from . import package_root, build_root, other_packages
from ._pep425 import _pep425tags, _pep425_implementation
if sys.version_info < (3,):
str_cls = unicode # noqa
else:
str_cls = str
def run():
"""
Installs required development dependencies. Uses git to checkout other
modularcrypto repos for more accurate coverage data.
"""
deps_dir = os.path.join(build_root, 'modularcrypto-deps')
if os.path.exists(deps_dir):
shutil.rmtree(deps_dir, ignore_errors=True)
os.mkdir(deps_dir)
try:
print("Staging ci dependencies")
_stage_requirements(deps_dir, os.path.join(package_root, 'requires', 'ci'))
print("Checking out modularcrypto packages for coverage")
for other_package in other_packages:
pkg_url = 'https://github.com/wbond/%s.git' % other_package
pkg_dir = os.path.join(build_root, other_package)
if os.path.exists(pkg_dir):
print("%s is already present" % other_package)
continue
print("Cloning %s" % pkg_url)
_execute(['git', 'clone', pkg_url], build_root)
print()
except (Exception):
if os.path.exists(deps_dir):
shutil.rmtree(deps_dir, ignore_errors=True)
raise
return True
def _download(url, dest):
"""
Downloads a URL to a directory
:param url:
The URL to download
:param dest:
The path to the directory to save the file in
:return:
The filesystem path to the saved file
"""
print('Downloading %s' % url)
filename = os.path.basename(url)
dest_path = os.path.join(dest, filename)
if sys.platform == 'win32':
powershell_exe = os.path.join('system32\\WindowsPowerShell\\v1.0\\powershell.exe')
code = "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12;"
code += "(New-Object Net.WebClient).DownloadFile('%s', '%s');" % (url, dest_path)
_execute([powershell_exe, '-Command', code], dest, 'Unable to connect to')
else:
_execute(
['curl', '-L', '--silent', '--show-error', '-O', url],
dest,
'Failed to connect to'
)
return dest_path
def _tuple_from_ver(version_string):
"""
:param version_string:
A unicode dotted version string
:return:
A tuple of integers
"""
match = re.search(
r'(\d+(?:\.\d+)*)'
r'([-._]?(?:alpha|a|beta|b|preview|pre|c|rc)\.?\d*)?'
r'(-\d+|(?:[-._]?(?:rev|r|post)\.?\d*))?'
r'([-._]?dev\.?\d*)?',
version_string
)
if not match:
return tuple()
nums = tuple(map(int, match.group(1).split('.')))
pre = match.group(2)
if pre:
pre = pre.replace('alpha', 'a')
pre = pre.replace('beta', 'b')
pre = pre.replace('preview', 'rc')
pre = pre.replace('pre', 'rc')
pre = re.sub(r'(?<!r)c', 'rc', pre)
pre = pre.lstrip('._-')
pre_dig_match = re.search(r'\d+', pre)
if pre_dig_match:
pre_dig = int(pre_dig_match.group(0))
else:
pre_dig = 0
pre = pre.rstrip('0123456789')
pre_num = {
'a': -3,
'b': -2,
'rc': -1,
}[pre]
pre_tup = (pre_num, pre_dig)
else:
pre_tup = tuple()
post = match.group(3)
if post:
post_dig_match = re.search(r'\d+', post)
if post_dig_match:
post_dig = int(post_dig_match.group(0))
else:
post_dig = 0
post_tup = (1, post_dig)
else:
post_tup = tuple()
dev = match.group(4)
if dev:
dev_dig_match = re.search(r'\d+', dev)
if dev_dig_match:
dev_dig = int(dev_dig_match.group(0))
else:
dev_dig = 0
dev_tup = (-4, dev_dig)
else:
dev_tup = tuple()
normalized = [nums]
if pre_tup:
normalized.append(pre_tup)
if post_tup:
normalized.append(post_tup)
if dev_tup:
normalized.append(dev_tup)
# This ensures regular releases happen after dev and prerelease, but
# before post releases
if not pre_tup and not post_tup and not dev_tup:
normalized.append((0, 0))
return tuple(normalized)
def _open_archive(path):
"""
:param path:
A unicode string of the filesystem path to the archive
:return:
An archive object
"""
if path.endswith('.zip'):
return zipfile.ZipFile(path, 'r')
return tarfile.open(path, 'r')
def _list_archive_members(archive):
"""
:param archive:
An archive from _open_archive()
:return:
A list of info objects to be used with _info_name() and _extract_info()
"""
if isinstance(archive, zipfile.ZipFile):
return archive.infolist()
return archive.getmembers()
def _archive_single_dir(archive):
"""
Check if all members of the archive are in a single top-level directory
:param archive:
An archive from _open_archive()
:return:
None if not a single top level directory in archive, otherwise a
unicode string of the top level directory name
"""
common_root = None
for info in _list_archive_members(archive):
fn = _info_name(info)
if fn in set(['.', '/']):
continue
sep = None
if '/' in fn:
sep = '/'
elif '\\' in fn:
sep = '\\'
if sep is None:
root_dir = fn
else:
root_dir, _ = fn.split(sep, 1)
if common_root is None:
common_root = root_dir
else:
if common_root != root_dir:
return None
return common_root
def _info_name(info):
"""
Returns a normalized file path for an archive info object
:param info:
An info object from _list_archive_members()
:return:
A unicode string with all directory separators normalized to "/"
"""
if isinstance(info, zipfile.ZipInfo):
return info.filename.replace('\\', '/')
return info.name.replace('\\', '/')
def _extract_info(archive, info):
"""
Extracts the contents of an archive info object
;param archive:
An archive from _open_archive()
:param info:
An info object from _list_archive_members()
:return:
None, or a byte string of the file contents
"""
if isinstance(archive, zipfile.ZipFile):
fn = info.filename
is_dir = fn.endswith('/') or fn.endswith('\\')
out = archive.read(info)
if is_dir and out == b'':
return None
return out
info_file = archive.extractfile(info)
if info_file:
return info_file.read()
return None
def _extract_package(deps_dir, pkg_path, pkg_dir):
"""
Extract a .whl, .zip, .tar.gz or .tar.bz2 into a package path to
use when running CI tasks
:param deps_dir:
A unicode string of the directory the package should be extracted to
:param pkg_path:
A unicode string of the path to the archive
:param pkg_dir:
If running setup.py, change to this dir first - a unicode string
"""
if pkg_path.endswith('.exe'):
try:
zf = None
zf = zipfile.ZipFile(pkg_path, 'r')
# Exes have a PLATLIB folder containing everything we want
for zi in zf.infolist():
if not zi.filename.startswith('PLATLIB'):
continue
data = _extract_info(zf, zi)
if data is not None:
dst_path = os.path.join(deps_dir, zi.filename[8:])
dst_dir = os.path.dirname(dst_path)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
with open(dst_path, 'wb') as f:
f.write(data)
finally:
if zf:
zf.close()
return
if pkg_path.endswith('.whl'):
try:
zf = None
zf = zipfile.ZipFile(pkg_path, 'r')
# Wheels contain exactly what we need and nothing else
zf.extractall(deps_dir)
finally:
if zf:
zf.close()
return
# Source archives may contain a bunch of other things, including mutliple
# packages, so we must use setup.py/setuptool to install/extract it
ar = None
staging_dir = os.path.join(deps_dir, '_staging')
try:
ar = _open_archive(pkg_path)
common_root = _archive_single_dir(ar)
members = []
for info in _list_archive_members(ar):
dst_rel_path = _info_name(info)
if common_root is not None:
dst_rel_path = dst_rel_path[len(common_root) + 1:]
members.append((info, dst_rel_path))
if not os.path.exists(staging_dir):
os.makedirs(staging_dir)
for info, rel_path in members:
info_data = _extract_info(ar, info)
# Dirs won't return a file
if info_data is not None:
dst_path = os.path.join(staging_dir, rel_path)
dst_dir = os.path.dirname(dst_path)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
with open(dst_path, 'wb') as f:
f.write(info_data)
setup_dir = staging_dir
if pkg_dir:
setup_dir = os.path.join(staging_dir, pkg_dir)
root = os.path.abspath(os.path.join(deps_dir, '..'))
install_lib = os.path.basename(deps_dir)
_execute(
[
sys.executable,
'setup.py',
'install',
'--root=%s' % root,
'--install-lib=%s' % install_lib,
'--no-compile'
],
setup_dir
)
finally:
if ar:
ar.close()
if staging_dir:
shutil.rmtree(staging_dir)
def _sort_pep440_versions(releases, include_prerelease):
"""
:param releases:
A list of unicode string PEP 440 version numbers
:param include_prerelease:
A boolean indicating if prerelease versions should be included
:return:
A sorted generator of 2-element tuples:
0: A unicode string containing a PEP 440 version number
1: A tuple of tuples containing integers - this is the output of
_tuple_from_ver() for the PEP 440 version number and is intended
for comparing versions
"""
parsed_versions = []
for v in releases:
t = _tuple_from_ver(v)
if not include_prerelease and t[1][0] < 0:
continue
parsed_versions.append((v, t))
return sorted(parsed_versions, key=lambda v: v[1])
def _is_valid_python_version(python_version, requires_python):
"""
Verifies the "python_version" and "requires_python" keys from a PyPi
download record are applicable to the current version of Python
:param python_version:
The "python_version" value from a PyPi download JSON structure. This
should be one of: "py2", "py3", "py2.py3" or "source".
:param requires_python:
The "requires_python" value from a PyPi download JSON structure. This
will be None, or a comma-separated list of conditions that must be
true. Ex: ">=3.5", "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7"
"""
if python_version == "py2" and sys.version_info >= (3,):
return False
if python_version == "py3" and sys.version_info < (3,):
return False
if requires_python is not None:
def _ver_tuples(ver_str):
ver_str = ver_str.strip()
if ver_str.endswith('.*'):
ver_str = ver_str[:-2]
cond_tup = tuple(map(int, ver_str.split('.')))
return (sys.version_info[:len(cond_tup)], cond_tup)
for part in map(str_cls.strip, requires_python.split(',')):
if part.startswith('!='):
sys_tup, cond_tup = _ver_tuples(part[2:])
if sys_tup == cond_tup:
return False
elif part.startswith('>='):
sys_tup, cond_tup = _ver_tuples(part[2:])
if sys_tup < cond_tup:
return False
elif part.startswith('>'):
sys_tup, cond_tup = _ver_tuples(part[1:])
if sys_tup <= cond_tup:
return False
elif part.startswith('<='):
sys_tup, cond_tup = _ver_tuples(part[2:])
if sys_tup > cond_tup:
return False
elif part.startswith('<'):
sys_tup, cond_tup = _ver_tuples(part[1:])
if sys_tup >= cond_tup:
return False
elif part.startswith('=='):
sys_tup, cond_tup = _ver_tuples(part[2:])
if sys_tup != cond_tup:
return False
return True
def _locate_suitable_download(downloads):
"""
:param downloads:
A list of dicts containing a key "url", "python_version" and
"requires_python"
:return:
A unicode string URL, or None if not a valid release for the current
version of Python
"""
valid_tags = _pep425tags()
exe_suffix = None
if sys.platform == 'win32' and _pep425_implementation() == 'cp':
win_arch = 'win32' if sys.maxsize == 2147483647 else 'win-amd64'
version_info = sys.version_info
exe_suffix = '.%s-py%d.%d.exe' % (win_arch, version_info[0], version_info[1])
wheels = {}
whl = None
tar_bz2 = None
tar_gz = None
exe = None
for download in downloads:
if not _is_valid_python_version(download.get('python_version'), download.get('requires_python')):
continue
if exe_suffix and download['url'].endswith(exe_suffix):
exe = download['url']
if download['url'].endswith('.whl'):
parts = os.path.basename(download['url']).split('-')
tag_impl = parts[-3]
tag_abi = parts[-2]
tag_arch = parts[-1].split('.')[0]
wheels[(tag_impl, tag_abi, tag_arch)] = download['url']
if download['url'].endswith('.tar.bz2'):
tar_bz2 = download['url']
if download['url'].endswith('.tar.gz'):
tar_gz = download['url']
# Find the most-specific wheel possible
for tag in valid_tags:
if tag in wheels:
whl = wheels[tag]
break
if exe_suffix and exe:
url = exe
elif whl:
url = whl
elif tar_bz2:
url = tar_bz2
elif tar_gz:
url = tar_gz
else:
return None
return url
def _stage_requirements(deps_dir, path):
"""
Installs requirements without using Python to download, since
different services are limiting to TLS 1.2, and older version of
Python do not support that
:param deps_dir:
A unicode path to a temporary diretory to use for downloads
:param path:
A unicode filesystem path to a requirements file
"""
packages = _parse_requires(path)
for p in packages:
url = None
pkg = p['pkg']
pkg_sub_dir = None
if p['type'] == 'url':
anchor = None
if '#' in pkg:
pkg, anchor = pkg.split('#', 1)
if '&' in anchor:
parts = anchor.split('&')
else:
parts = [anchor]
for part in parts:
param, value = part.split('=')
if param == 'subdirectory':
pkg_sub_dir = value
if pkg.endswith('.zip') or pkg.endswith('.tar.gz') or pkg.endswith('.tar.bz2') or pkg.endswith('.whl'):
url = pkg
else:
raise Exception('Unable to install package from URL that is not an archive')
else:
pypi_json_url = 'https://pypi.org/pypi/%s/json' % pkg
json_dest = _download(pypi_json_url, deps_dir)
with open(json_dest, 'rb') as f:
pkg_info = json.loads(f.read().decode('utf-8'))
if os.path.exists(json_dest):
os.remove(json_dest)
if p['type'] == '==':
if p['ver'] not in pkg_info['releases']:
raise Exception('Unable to find version %s of %s' % (p['ver'], pkg))
url = _locate_suitable_download(pkg_info['releases'][p['ver']])
if not url:
raise Exception('Unable to find a compatible download of %s == %s' % (pkg, p['ver']))
else:
p_ver_tup = _tuple_from_ver(p['ver'])
for ver_str, ver_tup in reversed(_sort_pep440_versions(pkg_info['releases'], False)):
if p['type'] == '>=' and ver_tup < p_ver_tup:
break
url = _locate_suitable_download(pkg_info['releases'][ver_str])
if url:
break
if not url:
if p['type'] == '>=':
raise Exception('Unable to find a compatible download of %s >= %s' % (pkg, p['ver']))
else:
raise Exception('Unable to find a compatible download of %s' % pkg)
local_path = _download(url, deps_dir)
_extract_package(deps_dir, local_path, pkg_sub_dir)
os.remove(local_path)
def _parse_requires(path):
"""
Does basic parsing of pip requirements files, to allow for
using something other than Python to do actual TLS requests
:param path:
A path to a requirements file
:return:
A list of dict objects containing the keys:
- 'type' ('any', 'url', '==', '>=')
- 'pkg'
- 'ver' (if 'type' == '==' or 'type' == '>=')
"""
python_version = '.'.join(map(str_cls, sys.version_info[0:2]))
sys_platform = sys.platform
packages = []
with open(path, 'rb') as f:
contents = f.read().decode('utf-8')
for line in re.split(r'\r?\n', contents):
line = line.strip()
if not len(line):
continue
if re.match(r'^\s*#', line):
continue
if ';' in line:
package, cond = line.split(';', 1)
package = package.strip()
cond = cond.strip()
cond = cond.replace('sys_platform', repr(sys_platform))
cond = cond.replace('python_version', repr(python_version))
if not eval(cond):
continue
else:
package = line.strip()
if re.match(r'^\s*-r\s*', package):
sub_req_file = re.sub(r'^\s*-r\s*', '', package)
sub_req_file = os.path.abspath(os.path.join(os.path.dirname(path), sub_req_file))
packages.extend(_parse_requires(sub_req_file))
continue
if re.match(r'https?://', package):
packages.append({'type': 'url', 'pkg': package})
continue
if '>=' in package:
parts = package.split('>=')
package = parts[0].strip()
ver = parts[1].strip()
packages.append({'type': '>=', 'pkg': package, 'ver': ver})
continue
if '==' in package:
parts = package.split('==')
package = parts[0].strip()
ver = parts[1].strip()
packages.append({'type': '==', 'pkg': package, 'ver': ver})
continue
if re.search(r'[^ a-zA-Z0-9\-]', package):
raise Exception('Unsupported requirements format version constraint: %s' % package)
packages.append({'type': 'any', 'pkg': package})
return packages
def _execute(params, cwd, retry=None):
"""
Executes a subprocess
:param params:
A list of the executable and arguments to pass to it
:param cwd:
The working directory to execute the command in
:param retry:
If this string is present in stderr, retry the operation
:return:
A 2-element tuple of (stdout, stderr)
"""
proc = subprocess.Popen(
params,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
cwd=cwd
)
stdout, stderr = proc.communicate()
code = proc.wait()
if code != 0:
if retry and retry in stderr.decode('utf-8'):
return _execute(params, cwd)
e = OSError('subprocess exit code for "%s" was %d: %s' % (' '.join(params), code, stderr))
e.stdout = stdout
e.stderr = stderr
raise e
return (stdout, stderr)
