1082 lines
43 KiB
Python
1082 lines
43 KiB
Python
# -*- coding: utf-8 -*-
|
|
#
|
|
# Copyright (C) 2013-2020 Vinay Sajip.
|
|
# Licensed to the Python Software Foundation under a contributor agreement.
|
|
# See LICENSE.txt and CONTRIBUTORS.txt.
|
|
#
|
|
from __future__ import unicode_literals
|
|
|
|
import base64
|
|
import codecs
|
|
import datetime
|
|
from email import message_from_file
|
|
import hashlib
|
|
import json
|
|
import logging
|
|
import os
|
|
import posixpath
|
|
import re
|
|
import shutil
|
|
import sys
|
|
import tempfile
|
|
import zipfile
|
|
|
|
from . import __version__, DistlibException
|
|
from .compat import sysconfig, ZipFile, fsdecode, text_type, filter
|
|
from .database import InstalledDistribution
|
|
from .metadata import (Metadata, METADATA_FILENAME, WHEEL_METADATA_FILENAME,
|
|
LEGACY_METADATA_FILENAME)
|
|
from .util import (FileOperator, convert_path, CSVReader, CSVWriter, Cache,
|
|
cached_property, get_cache_base, read_exports, tempdir,
|
|
get_platform)
|
|
from .version import NormalizedVersion, UnsupportedVersionError
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
cache = None # created when needed
|
|
|
|
if hasattr(sys, 'pypy_version_info'): # pragma: no cover
|
|
IMP_PREFIX = 'pp'
|
|
elif sys.platform.startswith('java'): # pragma: no cover
|
|
IMP_PREFIX = 'jy'
|
|
elif sys.platform == 'cli': # pragma: no cover
|
|
IMP_PREFIX = 'ip'
|
|
else:
|
|
IMP_PREFIX = 'cp'
|
|
|
|
VER_SUFFIX = sysconfig.get_config_var('py_version_nodot')
|
|
if not VER_SUFFIX: # pragma: no cover
|
|
VER_SUFFIX = '%s%s' % sys.version_info[:2]
|
|
PYVER = 'py' + VER_SUFFIX
|
|
IMPVER = IMP_PREFIX + VER_SUFFIX
|
|
|
|
ARCH = get_platform().replace('-', '_').replace('.', '_')
|
|
|
|
ABI = sysconfig.get_config_var('SOABI')
|
|
if ABI and ABI.startswith('cpython-'):
|
|
ABI = ABI.replace('cpython-', 'cp').split('-')[0]
|
|
else:
|
|
def _derive_abi():
|
|
parts = ['cp', VER_SUFFIX]
|
|
if sysconfig.get_config_var('Py_DEBUG'):
|
|
parts.append('d')
|
|
if IMP_PREFIX == 'cp':
|
|
vi = sys.version_info[:2]
|
|
if vi < (3, 8):
|
|
wpm = sysconfig.get_config_var('WITH_PYMALLOC')
|
|
if wpm is None:
|
|
wpm = True
|
|
if wpm:
|
|
parts.append('m')
|
|
if vi < (3, 3):
|
|
us = sysconfig.get_config_var('Py_UNICODE_SIZE')
|
|
if us == 4 or (us is None and sys.maxunicode == 0x10FFFF):
|
|
parts.append('u')
|
|
return ''.join(parts)
|
|
ABI = _derive_abi()
|
|
del _derive_abi
|
|
|
|
FILENAME_RE = re.compile(r'''
|
|
(?P<nm>[^-]+)
|
|
-(?P<vn>\d+[^-]*)
|
|
(-(?P<bn>\d+[^-]*))?
|
|
-(?P<py>\w+\d+(\.\w+\d+)*)
|
|
-(?P<bi>\w+)
|
|
-(?P<ar>\w+(\.\w+)*)
|
|
\.whl$
|
|
''', re.IGNORECASE | re.VERBOSE)
|
|
|
|
NAME_VERSION_RE = re.compile(r'''
|
|
(?P<nm>[^-]+)
|
|
-(?P<vn>\d+[^-]*)
|
|
(-(?P<bn>\d+[^-]*))?$
|
|
''', re.IGNORECASE | re.VERBOSE)
|
|
|
|
SHEBANG_RE = re.compile(br'\s*#![^\r\n]*')
|
|
SHEBANG_DETAIL_RE = re.compile(br'^(\s*#!("[^"]+"|\S+))\s+(.*)$')
|
|
SHEBANG_PYTHON = b'#!python'
|
|
SHEBANG_PYTHONW = b'#!pythonw'
|
|
|
|
if os.sep == '/':
|
|
to_posix = lambda o: o
|
|
else:
|
|
to_posix = lambda o: o.replace(os.sep, '/')
|
|
|
|
if sys.version_info[0] < 3:
|
|
import imp
|
|
else:
|
|
imp = None
|
|
import importlib.machinery
|
|
import importlib.util
|
|
|
|
def _get_suffixes():
|
|
if imp:
|
|
return [s[0] for s in imp.get_suffixes()]
|
|
else:
|
|
return importlib.machinery.EXTENSION_SUFFIXES
|
|
|
|
def _load_dynamic(name, path):
|
|
# https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly
|
|
if imp:
|
|
return imp.load_dynamic(name, path)
|
|
else:
|
|
spec = importlib.util.spec_from_file_location(name, path)
|
|
module = importlib.util.module_from_spec(spec)
|
|
sys.modules[name] = module
|
|
spec.loader.exec_module(module)
|
|
return module
|
|
|
|
class Mounter(object):
|
|
def __init__(self):
|
|
self.impure_wheels = {}
|
|
self.libs = {}
|
|
|
|
def add(self, pathname, extensions):
|
|
self.impure_wheels[pathname] = extensions
|
|
self.libs.update(extensions)
|
|
|
|
def remove(self, pathname):
|
|
extensions = self.impure_wheels.pop(pathname)
|
|
for k, v in extensions:
|
|
if k in self.libs:
|
|
del self.libs[k]
|
|
|
|
def find_module(self, fullname, path=None):
|
|
if fullname in self.libs:
|
|
result = self
|
|
else:
|
|
result = None
|
|
return result
|
|
|
|
def load_module(self, fullname):
|
|
if fullname in sys.modules:
|
|
result = sys.modules[fullname]
|
|
else:
|
|
if fullname not in self.libs:
|
|
raise ImportError('unable to find extension for %s' % fullname)
|
|
result = _load_dynamic(fullname, self.libs[fullname])
|
|
result.__loader__ = self
|
|
parts = fullname.rsplit('.', 1)
|
|
if len(parts) > 1:
|
|
result.__package__ = parts[0]
|
|
return result
|
|
|
|
_hook = Mounter()
|
|
|
|
|
|
class Wheel(object):
|
|
"""
|
|
Class to build and install from Wheel files (PEP 427).
|
|
"""
|
|
|
|
wheel_version = (1, 1)
|
|
hash_kind = 'sha256'
|
|
|
|
def __init__(self, filename=None, sign=False, verify=False):
|
|
"""
|
|
Initialise an instance using a (valid) filename.
|
|
"""
|
|
self.sign = sign
|
|
self.should_verify = verify
|
|
self.buildver = ''
|
|
self.pyver = [PYVER]
|
|
self.abi = ['none']
|
|
self.arch = ['any']
|
|
self.dirname = os.getcwd()
|
|
if filename is None:
|
|
self.name = 'dummy'
|
|
self.version = '0.1'
|
|
self._filename = self.filename
|
|
else:
|
|
m = NAME_VERSION_RE.match(filename)
|
|
if m:
|
|
info = m.groupdict('')
|
|
self.name = info['nm']
|
|
# Reinstate the local version separator
|
|
self.version = info['vn'].replace('_', '-')
|
|
self.buildver = info['bn']
|
|
self._filename = self.filename
|
|
else:
|
|
dirname, filename = os.path.split(filename)
|
|
m = FILENAME_RE.match(filename)
|
|
if not m:
|
|
raise DistlibException('Invalid name or '
|
|
'filename: %r' % filename)
|
|
if dirname:
|
|
self.dirname = os.path.abspath(dirname)
|
|
self._filename = filename
|
|
info = m.groupdict('')
|
|
self.name = info['nm']
|
|
self.version = info['vn']
|
|
self.buildver = info['bn']
|
|
self.pyver = info['py'].split('.')
|
|
self.abi = info['bi'].split('.')
|
|
self.arch = info['ar'].split('.')
|
|
|
|
@property
|
|
def filename(self):
|
|
"""
|
|
Build and return a filename from the various components.
|
|
"""
|
|
if self.buildver:
|
|
buildver = '-' + self.buildver
|
|
else:
|
|
buildver = ''
|
|
pyver = '.'.join(self.pyver)
|
|
abi = '.'.join(self.abi)
|
|
arch = '.'.join(self.arch)
|
|
# replace - with _ as a local version separator
|
|
version = self.version.replace('-', '_')
|
|
return '%s-%s%s-%s-%s-%s.whl' % (self.name, version, buildver,
|
|
pyver, abi, arch)
|
|
|
|
@property
|
|
def exists(self):
|
|
path = os.path.join(self.dirname, self.filename)
|
|
return os.path.isfile(path)
|
|
|
|
@property
|
|
def tags(self):
|
|
for pyver in self.pyver:
|
|
for abi in self.abi:
|
|
for arch in self.arch:
|
|
yield pyver, abi, arch
|
|
|
|
@cached_property
|
|
def metadata(self):
|
|
pathname = os.path.join(self.dirname, self.filename)
|
|
name_ver = '%s-%s' % (self.name, self.version)
|
|
info_dir = '%s.dist-info' % name_ver
|
|
wrapper = codecs.getreader('utf-8')
|
|
with ZipFile(pathname, 'r') as zf:
|
|
wheel_metadata = self.get_wheel_metadata(zf)
|
|
wv = wheel_metadata['Wheel-Version'].split('.', 1)
|
|
file_version = tuple([int(i) for i in wv])
|
|
# if file_version < (1, 1):
|
|
# fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME,
|
|
# LEGACY_METADATA_FILENAME]
|
|
# else:
|
|
# fns = [WHEEL_METADATA_FILENAME, METADATA_FILENAME]
|
|
fns = [WHEEL_METADATA_FILENAME, LEGACY_METADATA_FILENAME]
|
|
result = None
|
|
for fn in fns:
|
|
try:
|
|
metadata_filename = posixpath.join(info_dir, fn)
|
|
with zf.open(metadata_filename) as bf:
|
|
wf = wrapper(bf)
|
|
result = Metadata(fileobj=wf)
|
|
if result:
|
|
break
|
|
except KeyError:
|
|
pass
|
|
if not result:
|
|
raise ValueError('Invalid wheel, because metadata is '
|
|
'missing: looked in %s' % ', '.join(fns))
|
|
return result
|
|
|
|
def get_wheel_metadata(self, zf):
|
|
name_ver = '%s-%s' % (self.name, self.version)
|
|
info_dir = '%s.dist-info' % name_ver
|
|
metadata_filename = posixpath.join(info_dir, 'WHEEL')
|
|
with zf.open(metadata_filename) as bf:
|
|
wf = codecs.getreader('utf-8')(bf)
|
|
message = message_from_file(wf)
|
|
return dict(message)
|
|
|
|
@cached_property
|
|
def info(self):
|
|
pathname = os.path.join(self.dirname, self.filename)
|
|
with ZipFile(pathname, 'r') as zf:
|
|
result = self.get_wheel_metadata(zf)
|
|
return result
|
|
|
|
def process_shebang(self, data):
|
|
m = SHEBANG_RE.match(data)
|
|
if m:
|
|
end = m.end()
|
|
shebang, data_after_shebang = data[:end], data[end:]
|
|
# Preserve any arguments after the interpreter
|
|
if b'pythonw' in shebang.lower():
|
|
shebang_python = SHEBANG_PYTHONW
|
|
else:
|
|
shebang_python = SHEBANG_PYTHON
|
|
m = SHEBANG_DETAIL_RE.match(shebang)
|
|
if m:
|
|
args = b' ' + m.groups()[-1]
|
|
else:
|
|
args = b''
|
|
shebang = shebang_python + args
|
|
data = shebang + data_after_shebang
|
|
else:
|
|
cr = data.find(b'\r')
|
|
lf = data.find(b'\n')
|
|
if cr < 0 or cr > lf:
|
|
term = b'\n'
|
|
else:
|
|
if data[cr:cr + 2] == b'\r\n':
|
|
term = b'\r\n'
|
|
else:
|
|
term = b'\r'
|
|
data = SHEBANG_PYTHON + term + data
|
|
return data
|
|
|
|
def get_hash(self, data, hash_kind=None):
|
|
if hash_kind is None:
|
|
hash_kind = self.hash_kind
|
|
try:
|
|
hasher = getattr(hashlib, hash_kind)
|
|
except AttributeError:
|
|
raise DistlibException('Unsupported hash algorithm: %r' % hash_kind)
|
|
result = hasher(data).digest()
|
|
result = base64.urlsafe_b64encode(result).rstrip(b'=').decode('ascii')
|
|
return hash_kind, result
|
|
|
|
def write_record(self, records, record_path, archive_record_path):
|
|
records = list(records) # make a copy, as mutated
|
|
records.append((archive_record_path, '', ''))
|
|
with CSVWriter(record_path) as writer:
|
|
for row in records:
|
|
writer.writerow(row)
|
|
|
|
def write_records(self, info, libdir, archive_paths):
|
|
records = []
|
|
distinfo, info_dir = info
|
|
hasher = getattr(hashlib, self.hash_kind)
|
|
for ap, p in archive_paths:
|
|
with open(p, 'rb') as f:
|
|
data = f.read()
|
|
digest = '%s=%s' % self.get_hash(data)
|
|
size = os.path.getsize(p)
|
|
records.append((ap, digest, size))
|
|
|
|
p = os.path.join(distinfo, 'RECORD')
|
|
ap = to_posix(os.path.join(info_dir, 'RECORD'))
|
|
self.write_record(records, p, ap)
|
|
archive_paths.append((ap, p))
|
|
|
|
def build_zip(self, pathname, archive_paths):
|
|
with ZipFile(pathname, 'w', zipfile.ZIP_DEFLATED) as zf:
|
|
for ap, p in archive_paths:
|
|
logger.debug('Wrote %s to %s in wheel', p, ap)
|
|
zf.write(p, ap)
|
|
|
|
def build(self, paths, tags=None, wheel_version=None):
|
|
"""
|
|
Build a wheel from files in specified paths, and use any specified tags
|
|
when determining the name of the wheel.
|
|
"""
|
|
if tags is None:
|
|
tags = {}
|
|
|
|
libkey = list(filter(lambda o: o in paths, ('purelib', 'platlib')))[0]
|
|
if libkey == 'platlib':
|
|
is_pure = 'false'
|
|
default_pyver = [IMPVER]
|
|
default_abi = [ABI]
|
|
default_arch = [ARCH]
|
|
else:
|
|
is_pure = 'true'
|
|
default_pyver = [PYVER]
|
|
default_abi = ['none']
|
|
default_arch = ['any']
|
|
|
|
self.pyver = tags.get('pyver', default_pyver)
|
|
self.abi = tags.get('abi', default_abi)
|
|
self.arch = tags.get('arch', default_arch)
|
|
|
|
libdir = paths[libkey]
|
|
|
|
name_ver = '%s-%s' % (self.name, self.version)
|
|
data_dir = '%s.data' % name_ver
|
|
info_dir = '%s.dist-info' % name_ver
|
|
|
|
archive_paths = []
|
|
|
|
# First, stuff which is not in site-packages
|
|
for key in ('data', 'headers', 'scripts'):
|
|
if key not in paths:
|
|
continue
|
|
path = paths[key]
|
|
if os.path.isdir(path):
|
|
for root, dirs, files in os.walk(path):
|
|
for fn in files:
|
|
p = fsdecode(os.path.join(root, fn))
|
|
rp = os.path.relpath(p, path)
|
|
ap = to_posix(os.path.join(data_dir, key, rp))
|
|
archive_paths.append((ap, p))
|
|
if key == 'scripts' and not p.endswith('.exe'):
|
|
with open(p, 'rb') as f:
|
|
data = f.read()
|
|
data = self.process_shebang(data)
|
|
with open(p, 'wb') as f:
|
|
f.write(data)
|
|
|
|
# Now, stuff which is in site-packages, other than the
|
|
# distinfo stuff.
|
|
path = libdir
|
|
distinfo = None
|
|
for root, dirs, files in os.walk(path):
|
|
if root == path:
|
|
# At the top level only, save distinfo for later
|
|
# and skip it for now
|
|
for i, dn in enumerate(dirs):
|
|
dn = fsdecode(dn)
|
|
if dn.endswith('.dist-info'):
|
|
distinfo = os.path.join(root, dn)
|
|
del dirs[i]
|
|
break
|
|
assert distinfo, '.dist-info directory expected, not found'
|
|
|
|
for fn in files:
|
|
# comment out next suite to leave .pyc files in
|
|
if fsdecode(fn).endswith(('.pyc', '.pyo')):
|
|
continue
|
|
p = os.path.join(root, fn)
|
|
rp = to_posix(os.path.relpath(p, path))
|
|
archive_paths.append((rp, p))
|
|
|
|
# Now distinfo. Assumed to be flat, i.e. os.listdir is enough.
|
|
files = os.listdir(distinfo)
|
|
for fn in files:
|
|
if fn not in ('RECORD', 'INSTALLER', 'SHARED', 'WHEEL'):
|
|
p = fsdecode(os.path.join(distinfo, fn))
|
|
ap = to_posix(os.path.join(info_dir, fn))
|
|
archive_paths.append((ap, p))
|
|
|
|
wheel_metadata = [
|
|
'Wheel-Version: %d.%d' % (wheel_version or self.wheel_version),
|
|
'Generator: distlib %s' % __version__,
|
|
'Root-Is-Purelib: %s' % is_pure,
|
|
]
|
|
for pyver, abi, arch in self.tags:
|
|
wheel_metadata.append('Tag: %s-%s-%s' % (pyver, abi, arch))
|
|
p = os.path.join(distinfo, 'WHEEL')
|
|
with open(p, 'w') as f:
|
|
f.write('\n'.join(wheel_metadata))
|
|
ap = to_posix(os.path.join(info_dir, 'WHEEL'))
|
|
archive_paths.append((ap, p))
|
|
|
|
# sort the entries by archive path. Not needed by any spec, but it
|
|
# keeps the archive listing and RECORD tidier than they would otherwise
|
|
# be. Use the number of path segments to keep directory entries together,
|
|
# and keep the dist-info stuff at the end.
|
|
def sorter(t):
|
|
ap = t[0]
|
|
n = ap.count('/')
|
|
if '.dist-info' in ap:
|
|
n += 10000
|
|
return (n, ap)
|
|
archive_paths = sorted(archive_paths, key=sorter)
|
|
|
|
# Now, at last, RECORD.
|
|
# Paths in here are archive paths - nothing else makes sense.
|
|
self.write_records((distinfo, info_dir), libdir, archive_paths)
|
|
# Now, ready to build the zip file
|
|
pathname = os.path.join(self.dirname, self.filename)
|
|
self.build_zip(pathname, archive_paths)
|
|
return pathname
|
|
|
|
def skip_entry(self, arcname):
|
|
"""
|
|
Determine whether an archive entry should be skipped when verifying
|
|
or installing.
|
|
"""
|
|
# The signature file won't be in RECORD,
|
|
# and we don't currently don't do anything with it
|
|
# We also skip directories, as they won't be in RECORD
|
|
# either. See:
|
|
#
|
|
# https://github.com/pypa/wheel/issues/294
|
|
# https://github.com/pypa/wheel/issues/287
|
|
# https://github.com/pypa/wheel/pull/289
|
|
#
|
|
return arcname.endswith(('/', '/RECORD.jws'))
|
|
|
|
def install(self, paths, maker, **kwargs):
|
|
"""
|
|
Install a wheel to the specified paths. If kwarg ``warner`` is
|
|
specified, it should be a callable, which will be called with two
|
|
tuples indicating the wheel version of this software and the wheel
|
|
version in the file, if there is a discrepancy in the versions.
|
|
This can be used to issue any warnings to raise any exceptions.
|
|
If kwarg ``lib_only`` is True, only the purelib/platlib files are
|
|
installed, and the headers, scripts, data and dist-info metadata are
|
|
not written. If kwarg ``bytecode_hashed_invalidation`` is True, written
|
|
bytecode will try to use file-hash based invalidation (PEP-552) on
|
|
supported interpreter versions (CPython 2.7+).
|
|
|
|
The return value is a :class:`InstalledDistribution` instance unless
|
|
``options.lib_only`` is True, in which case the return value is ``None``.
|
|
"""
|
|
|
|
dry_run = maker.dry_run
|
|
warner = kwargs.get('warner')
|
|
lib_only = kwargs.get('lib_only', False)
|
|
bc_hashed_invalidation = kwargs.get('bytecode_hashed_invalidation', False)
|
|
|
|
pathname = os.path.join(self.dirname, self.filename)
|
|
name_ver = '%s-%s' % (self.name, self.version)
|
|
data_dir = '%s.data' % name_ver
|
|
info_dir = '%s.dist-info' % name_ver
|
|
|
|
metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME)
|
|
wheel_metadata_name = posixpath.join(info_dir, 'WHEEL')
|
|
record_name = posixpath.join(info_dir, 'RECORD')
|
|
|
|
wrapper = codecs.getreader('utf-8')
|
|
|
|
with ZipFile(pathname, 'r') as zf:
|
|
with zf.open(wheel_metadata_name) as bwf:
|
|
wf = wrapper(bwf)
|
|
message = message_from_file(wf)
|
|
wv = message['Wheel-Version'].split('.', 1)
|
|
file_version = tuple([int(i) for i in wv])
|
|
if (file_version != self.wheel_version) and warner:
|
|
warner(self.wheel_version, file_version)
|
|
|
|
if message['Root-Is-Purelib'] == 'true':
|
|
libdir = paths['purelib']
|
|
else:
|
|
libdir = paths['platlib']
|
|
|
|
records = {}
|
|
with zf.open(record_name) as bf:
|
|
with CSVReader(stream=bf) as reader:
|
|
for row in reader:
|
|
p = row[0]
|
|
records[p] = row
|
|
|
|
data_pfx = posixpath.join(data_dir, '')
|
|
info_pfx = posixpath.join(info_dir, '')
|
|
script_pfx = posixpath.join(data_dir, 'scripts', '')
|
|
|
|
# make a new instance rather than a copy of maker's,
|
|
# as we mutate it
|
|
fileop = FileOperator(dry_run=dry_run)
|
|
fileop.record = True # so we can rollback if needed
|
|
|
|
bc = not sys.dont_write_bytecode # Double negatives. Lovely!
|
|
|
|
outfiles = [] # for RECORD writing
|
|
|
|
# for script copying/shebang processing
|
|
workdir = tempfile.mkdtemp()
|
|
# set target dir later
|
|
# we default add_launchers to False, as the
|
|
# Python Launcher should be used instead
|
|
maker.source_dir = workdir
|
|
maker.target_dir = None
|
|
try:
|
|
for zinfo in zf.infolist():
|
|
arcname = zinfo.filename
|
|
if isinstance(arcname, text_type):
|
|
u_arcname = arcname
|
|
else:
|
|
u_arcname = arcname.decode('utf-8')
|
|
if self.skip_entry(u_arcname):
|
|
continue
|
|
row = records[u_arcname]
|
|
if row[2] and str(zinfo.file_size) != row[2]:
|
|
raise DistlibException('size mismatch for '
|
|
'%s' % u_arcname)
|
|
if row[1]:
|
|
kind, value = row[1].split('=', 1)
|
|
with zf.open(arcname) as bf:
|
|
data = bf.read()
|
|
_, digest = self.get_hash(data, kind)
|
|
if digest != value:
|
|
raise DistlibException('digest mismatch for '
|
|
'%s' % arcname)
|
|
|
|
if lib_only and u_arcname.startswith((info_pfx, data_pfx)):
|
|
logger.debug('lib_only: skipping %s', u_arcname)
|
|
continue
|
|
is_script = (u_arcname.startswith(script_pfx)
|
|
and not u_arcname.endswith('.exe'))
|
|
|
|
if u_arcname.startswith(data_pfx):
|
|
_, where, rp = u_arcname.split('/', 2)
|
|
outfile = os.path.join(paths[where], convert_path(rp))
|
|
else:
|
|
# meant for site-packages.
|
|
if u_arcname in (wheel_metadata_name, record_name):
|
|
continue
|
|
outfile = os.path.join(libdir, convert_path(u_arcname))
|
|
if not is_script:
|
|
with zf.open(arcname) as bf:
|
|
fileop.copy_stream(bf, outfile)
|
|
# Issue #147: permission bits aren't preserved. Using
|
|
# zf.extract(zinfo, libdir) should have worked, but didn't,
|
|
# see https://www.thetopsites.net/article/53834422.shtml
|
|
# So ... manually preserve permission bits as given in zinfo
|
|
if os.name == 'posix':
|
|
# just set the normal permission bits
|
|
os.chmod(outfile, (zinfo.external_attr >> 16) & 0x1FF)
|
|
outfiles.append(outfile)
|
|
# Double check the digest of the written file
|
|
if not dry_run and row[1]:
|
|
with open(outfile, 'rb') as bf:
|
|
data = bf.read()
|
|
_, newdigest = self.get_hash(data, kind)
|
|
if newdigest != digest:
|
|
raise DistlibException('digest mismatch '
|
|
'on write for '
|
|
'%s' % outfile)
|
|
if bc and outfile.endswith('.py'):
|
|
try:
|
|
pyc = fileop.byte_compile(outfile,
|
|
hashed_invalidation=bc_hashed_invalidation)
|
|
outfiles.append(pyc)
|
|
except Exception:
|
|
# Don't give up if byte-compilation fails,
|
|
# but log it and perhaps warn the user
|
|
logger.warning('Byte-compilation failed',
|
|
exc_info=True)
|
|
else:
|
|
fn = os.path.basename(convert_path(arcname))
|
|
workname = os.path.join(workdir, fn)
|
|
with zf.open(arcname) as bf:
|
|
fileop.copy_stream(bf, workname)
|
|
|
|
dn, fn = os.path.split(outfile)
|
|
maker.target_dir = dn
|
|
filenames = maker.make(fn)
|
|
fileop.set_executable_mode(filenames)
|
|
outfiles.extend(filenames)
|
|
|
|
if lib_only:
|
|
logger.debug('lib_only: returning None')
|
|
dist = None
|
|
else:
|
|
# Generate scripts
|
|
|
|
# Try to get pydist.json so we can see if there are
|
|
# any commands to generate. If this fails (e.g. because
|
|
# of a legacy wheel), log a warning but don't give up.
|
|
commands = None
|
|
file_version = self.info['Wheel-Version']
|
|
if file_version == '1.0':
|
|
# Use legacy info
|
|
ep = posixpath.join(info_dir, 'entry_points.txt')
|
|
try:
|
|
with zf.open(ep) as bwf:
|
|
epdata = read_exports(bwf)
|
|
commands = {}
|
|
for key in ('console', 'gui'):
|
|
k = '%s_scripts' % key
|
|
if k in epdata:
|
|
commands['wrap_%s' % key] = d = {}
|
|
for v in epdata[k].values():
|
|
s = '%s:%s' % (v.prefix, v.suffix)
|
|
if v.flags:
|
|
s += ' [%s]' % ','.join(v.flags)
|
|
d[v.name] = s
|
|
except Exception:
|
|
logger.warning('Unable to read legacy script '
|
|
'metadata, so cannot generate '
|
|
'scripts')
|
|
else:
|
|
try:
|
|
with zf.open(metadata_name) as bwf:
|
|
wf = wrapper(bwf)
|
|
commands = json.load(wf).get('extensions')
|
|
if commands:
|
|
commands = commands.get('python.commands')
|
|
except Exception:
|
|
logger.warning('Unable to read JSON metadata, so '
|
|
'cannot generate scripts')
|
|
if commands:
|
|
console_scripts = commands.get('wrap_console', {})
|
|
gui_scripts = commands.get('wrap_gui', {})
|
|
if console_scripts or gui_scripts:
|
|
script_dir = paths.get('scripts', '')
|
|
if not os.path.isdir(script_dir):
|
|
raise ValueError('Valid script path not '
|
|
'specified')
|
|
maker.target_dir = script_dir
|
|
for k, v in console_scripts.items():
|
|
script = '%s = %s' % (k, v)
|
|
filenames = maker.make(script)
|
|
fileop.set_executable_mode(filenames)
|
|
|
|
if gui_scripts:
|
|
options = {'gui': True }
|
|
for k, v in gui_scripts.items():
|
|
script = '%s = %s' % (k, v)
|
|
filenames = maker.make(script, options)
|
|
fileop.set_executable_mode(filenames)
|
|
|
|
p = os.path.join(libdir, info_dir)
|
|
dist = InstalledDistribution(p)
|
|
|
|
# Write SHARED
|
|
paths = dict(paths) # don't change passed in dict
|
|
del paths['purelib']
|
|
del paths['platlib']
|
|
paths['lib'] = libdir
|
|
p = dist.write_shared_locations(paths, dry_run)
|
|
if p:
|
|
outfiles.append(p)
|
|
|
|
# Write RECORD
|
|
dist.write_installed_files(outfiles, paths['prefix'],
|
|
dry_run)
|
|
return dist
|
|
except Exception: # pragma: no cover
|
|
logger.exception('installation failed.')
|
|
fileop.rollback()
|
|
raise
|
|
finally:
|
|
shutil.rmtree(workdir)
|
|
|
|
def _get_dylib_cache(self):
|
|
global cache
|
|
if cache is None:
|
|
# Use native string to avoid issues on 2.x: see Python #20140.
|
|
base = os.path.join(get_cache_base(), str('dylib-cache'),
|
|
'%s.%s' % sys.version_info[:2])
|
|
cache = Cache(base)
|
|
return cache
|
|
|
|
def _get_extensions(self):
|
|
pathname = os.path.join(self.dirname, self.filename)
|
|
name_ver = '%s-%s' % (self.name, self.version)
|
|
info_dir = '%s.dist-info' % name_ver
|
|
arcname = posixpath.join(info_dir, 'EXTENSIONS')
|
|
wrapper = codecs.getreader('utf-8')
|
|
result = []
|
|
with ZipFile(pathname, 'r') as zf:
|
|
try:
|
|
with zf.open(arcname) as bf:
|
|
wf = wrapper(bf)
|
|
extensions = json.load(wf)
|
|
cache = self._get_dylib_cache()
|
|
prefix = cache.prefix_to_dir(pathname)
|
|
cache_base = os.path.join(cache.base, prefix)
|
|
if not os.path.isdir(cache_base):
|
|
os.makedirs(cache_base)
|
|
for name, relpath in extensions.items():
|
|
dest = os.path.join(cache_base, convert_path(relpath))
|
|
if not os.path.exists(dest):
|
|
extract = True
|
|
else:
|
|
file_time = os.stat(dest).st_mtime
|
|
file_time = datetime.datetime.fromtimestamp(file_time)
|
|
info = zf.getinfo(relpath)
|
|
wheel_time = datetime.datetime(*info.date_time)
|
|
extract = wheel_time > file_time
|
|
if extract:
|
|
zf.extract(relpath, cache_base)
|
|
result.append((name, dest))
|
|
except KeyError:
|
|
pass
|
|
return result
|
|
|
|
def is_compatible(self):
|
|
"""
|
|
Determine if a wheel is compatible with the running system.
|
|
"""
|
|
return is_compatible(self)
|
|
|
|
def is_mountable(self):
|
|
"""
|
|
Determine if a wheel is asserted as mountable by its metadata.
|
|
"""
|
|
return True # for now - metadata details TBD
|
|
|
|
def mount(self, append=False):
|
|
pathname = os.path.abspath(os.path.join(self.dirname, self.filename))
|
|
if not self.is_compatible():
|
|
msg = 'Wheel %s not compatible with this Python.' % pathname
|
|
raise DistlibException(msg)
|
|
if not self.is_mountable():
|
|
msg = 'Wheel %s is marked as not mountable.' % pathname
|
|
raise DistlibException(msg)
|
|
if pathname in sys.path:
|
|
logger.debug('%s already in path', pathname)
|
|
else:
|
|
if append:
|
|
sys.path.append(pathname)
|
|
else:
|
|
sys.path.insert(0, pathname)
|
|
extensions = self._get_extensions()
|
|
if extensions:
|
|
if _hook not in sys.meta_path:
|
|
sys.meta_path.append(_hook)
|
|
_hook.add(pathname, extensions)
|
|
|
|
def unmount(self):
|
|
pathname = os.path.abspath(os.path.join(self.dirname, self.filename))
|
|
if pathname not in sys.path:
|
|
logger.debug('%s not in path', pathname)
|
|
else:
|
|
sys.path.remove(pathname)
|
|
if pathname in _hook.impure_wheels:
|
|
_hook.remove(pathname)
|
|
if not _hook.impure_wheels:
|
|
if _hook in sys.meta_path:
|
|
sys.meta_path.remove(_hook)
|
|
|
|
def verify(self):
|
|
pathname = os.path.join(self.dirname, self.filename)
|
|
name_ver = '%s-%s' % (self.name, self.version)
|
|
data_dir = '%s.data' % name_ver
|
|
info_dir = '%s.dist-info' % name_ver
|
|
|
|
metadata_name = posixpath.join(info_dir, LEGACY_METADATA_FILENAME)
|
|
wheel_metadata_name = posixpath.join(info_dir, 'WHEEL')
|
|
record_name = posixpath.join(info_dir, 'RECORD')
|
|
|
|
wrapper = codecs.getreader('utf-8')
|
|
|
|
with ZipFile(pathname, 'r') as zf:
|
|
with zf.open(wheel_metadata_name) as bwf:
|
|
wf = wrapper(bwf)
|
|
message = message_from_file(wf)
|
|
wv = message['Wheel-Version'].split('.', 1)
|
|
file_version = tuple([int(i) for i in wv])
|
|
# TODO version verification
|
|
|
|
records = {}
|
|
with zf.open(record_name) as bf:
|
|
with CSVReader(stream=bf) as reader:
|
|
for row in reader:
|
|
p = row[0]
|
|
records[p] = row
|
|
|
|
for zinfo in zf.infolist():
|
|
arcname = zinfo.filename
|
|
if isinstance(arcname, text_type):
|
|
u_arcname = arcname
|
|
else:
|
|
u_arcname = arcname.decode('utf-8')
|
|
# See issue #115: some wheels have .. in their entries, but
|
|
# in the filename ... e.g. __main__..py ! So the check is
|
|
# updated to look for .. in the directory portions
|
|
p = u_arcname.split('/')
|
|
if '..' in p:
|
|
raise DistlibException('invalid entry in '
|
|
'wheel: %r' % u_arcname)
|
|
|
|
if self.skip_entry(u_arcname):
|
|
continue
|
|
row = records[u_arcname]
|
|
if row[2] and str(zinfo.file_size) != row[2]:
|
|
raise DistlibException('size mismatch for '
|
|
'%s' % u_arcname)
|
|
if row[1]:
|
|
kind, value = row[1].split('=', 1)
|
|
with zf.open(arcname) as bf:
|
|
data = bf.read()
|
|
_, digest = self.get_hash(data, kind)
|
|
if digest != value:
|
|
raise DistlibException('digest mismatch for '
|
|
'%s' % arcname)
|
|
|
|
def update(self, modifier, dest_dir=None, **kwargs):
|
|
"""
|
|
Update the contents of a wheel in a generic way. The modifier should
|
|
be a callable which expects a dictionary argument: its keys are
|
|
archive-entry paths, and its values are absolute filesystem paths
|
|
where the contents the corresponding archive entries can be found. The
|
|
modifier is free to change the contents of the files pointed to, add
|
|
new entries and remove entries, before returning. This method will
|
|
extract the entire contents of the wheel to a temporary location, call
|
|
the modifier, and then use the passed (and possibly updated)
|
|
dictionary to write a new wheel. If ``dest_dir`` is specified, the new
|
|
wheel is written there -- otherwise, the original wheel is overwritten.
|
|
|
|
The modifier should return True if it updated the wheel, else False.
|
|
This method returns the same value the modifier returns.
|
|
"""
|
|
|
|
def get_version(path_map, info_dir):
|
|
version = path = None
|
|
key = '%s/%s' % (info_dir, LEGACY_METADATA_FILENAME)
|
|
if key not in path_map:
|
|
key = '%s/PKG-INFO' % info_dir
|
|
if key in path_map:
|
|
path = path_map[key]
|
|
version = Metadata(path=path).version
|
|
return version, path
|
|
|
|
def update_version(version, path):
|
|
updated = None
|
|
try:
|
|
v = NormalizedVersion(version)
|
|
i = version.find('-')
|
|
if i < 0:
|
|
updated = '%s+1' % version
|
|
else:
|
|
parts = [int(s) for s in version[i + 1:].split('.')]
|
|
parts[-1] += 1
|
|
updated = '%s+%s' % (version[:i],
|
|
'.'.join(str(i) for i in parts))
|
|
except UnsupportedVersionError:
|
|
logger.debug('Cannot update non-compliant (PEP-440) '
|
|
'version %r', version)
|
|
if updated:
|
|
md = Metadata(path=path)
|
|
md.version = updated
|
|
legacy = path.endswith(LEGACY_METADATA_FILENAME)
|
|
md.write(path=path, legacy=legacy)
|
|
logger.debug('Version updated from %r to %r', version,
|
|
updated)
|
|
|
|
pathname = os.path.join(self.dirname, self.filename)
|
|
name_ver = '%s-%s' % (self.name, self.version)
|
|
info_dir = '%s.dist-info' % name_ver
|
|
record_name = posixpath.join(info_dir, 'RECORD')
|
|
with tempdir() as workdir:
|
|
with ZipFile(pathname, 'r') as zf:
|
|
path_map = {}
|
|
for zinfo in zf.infolist():
|
|
arcname = zinfo.filename
|
|
if isinstance(arcname, text_type):
|
|
u_arcname = arcname
|
|
else:
|
|
u_arcname = arcname.decode('utf-8')
|
|
if u_arcname == record_name:
|
|
continue
|
|
if '..' in u_arcname:
|
|
raise DistlibException('invalid entry in '
|
|
'wheel: %r' % u_arcname)
|
|
zf.extract(zinfo, workdir)
|
|
path = os.path.join(workdir, convert_path(u_arcname))
|
|
path_map[u_arcname] = path
|
|
|
|
# Remember the version.
|
|
original_version, _ = get_version(path_map, info_dir)
|
|
# Files extracted. Call the modifier.
|
|
modified = modifier(path_map, **kwargs)
|
|
if modified:
|
|
# Something changed - need to build a new wheel.
|
|
current_version, path = get_version(path_map, info_dir)
|
|
if current_version and (current_version == original_version):
|
|
# Add or update local version to signify changes.
|
|
update_version(current_version, path)
|
|
# Decide where the new wheel goes.
|
|
if dest_dir is None:
|
|
fd, newpath = tempfile.mkstemp(suffix='.whl',
|
|
prefix='wheel-update-',
|
|
dir=workdir)
|
|
os.close(fd)
|
|
else:
|
|
if not os.path.isdir(dest_dir):
|
|
raise DistlibException('Not a directory: %r' % dest_dir)
|
|
newpath = os.path.join(dest_dir, self.filename)
|
|
archive_paths = list(path_map.items())
|
|
distinfo = os.path.join(workdir, info_dir)
|
|
info = distinfo, info_dir
|
|
self.write_records(info, workdir, archive_paths)
|
|
self.build_zip(newpath, archive_paths)
|
|
if dest_dir is None:
|
|
shutil.copyfile(newpath, pathname)
|
|
return modified
|
|
|
|
def _get_glibc_version():
|
|
import platform
|
|
ver = platform.libc_ver()
|
|
result = []
|
|
if ver[0] == 'glibc':
|
|
for s in ver[1].split('.'):
|
|
result.append(int(s) if s.isdigit() else 0)
|
|
result = tuple(result)
|
|
return result
|
|
|
|
def compatible_tags():
|
|
"""
|
|
Return (pyver, abi, arch) tuples compatible with this Python.
|
|
"""
|
|
versions = [VER_SUFFIX]
|
|
major = VER_SUFFIX[0]
|
|
for minor in range(sys.version_info[1] - 1, - 1, -1):
|
|
versions.append(''.join([major, str(minor)]))
|
|
|
|
abis = []
|
|
for suffix in _get_suffixes():
|
|
if suffix.startswith('.abi'):
|
|
abis.append(suffix.split('.', 2)[1])
|
|
abis.sort()
|
|
if ABI != 'none':
|
|
abis.insert(0, ABI)
|
|
abis.append('none')
|
|
result = []
|
|
|
|
arches = [ARCH]
|
|
if sys.platform == 'darwin':
|
|
m = re.match(r'(\w+)_(\d+)_(\d+)_(\w+)$', ARCH)
|
|
if m:
|
|
name, major, minor, arch = m.groups()
|
|
minor = int(minor)
|
|
matches = [arch]
|
|
if arch in ('i386', 'ppc'):
|
|
matches.append('fat')
|
|
if arch in ('i386', 'ppc', 'x86_64'):
|
|
matches.append('fat3')
|
|
if arch in ('ppc64', 'x86_64'):
|
|
matches.append('fat64')
|
|
if arch in ('i386', 'x86_64'):
|
|
matches.append('intel')
|
|
if arch in ('i386', 'x86_64', 'intel', 'ppc', 'ppc64'):
|
|
matches.append('universal')
|
|
while minor >= 0:
|
|
for match in matches:
|
|
s = '%s_%s_%s_%s' % (name, major, minor, match)
|
|
if s != ARCH: # already there
|
|
arches.append(s)
|
|
minor -= 1
|
|
|
|
# Most specific - our Python version, ABI and arch
|
|
for abi in abis:
|
|
for arch in arches:
|
|
result.append((''.join((IMP_PREFIX, versions[0])), abi, arch))
|
|
# manylinux
|
|
if abi != 'none' and sys.platform.startswith('linux'):
|
|
arch = arch.replace('linux_', '')
|
|
parts = _get_glibc_version()
|
|
if len(parts) == 2:
|
|
if parts >= (2, 5):
|
|
result.append((''.join((IMP_PREFIX, versions[0])), abi,
|
|
'manylinux1_%s' % arch))
|
|
if parts >= (2, 12):
|
|
result.append((''.join((IMP_PREFIX, versions[0])), abi,
|
|
'manylinux2010_%s' % arch))
|
|
if parts >= (2, 17):
|
|
result.append((''.join((IMP_PREFIX, versions[0])), abi,
|
|
'manylinux2014_%s' % arch))
|
|
result.append((''.join((IMP_PREFIX, versions[0])), abi,
|
|
'manylinux_%s_%s_%s' % (parts[0], parts[1],
|
|
arch)))
|
|
|
|
# where no ABI / arch dependency, but IMP_PREFIX dependency
|
|
for i, version in enumerate(versions):
|
|
result.append((''.join((IMP_PREFIX, version)), 'none', 'any'))
|
|
if i == 0:
|
|
result.append((''.join((IMP_PREFIX, version[0])), 'none', 'any'))
|
|
|
|
# no IMP_PREFIX, ABI or arch dependency
|
|
for i, version in enumerate(versions):
|
|
result.append((''.join(('py', version)), 'none', 'any'))
|
|
if i == 0:
|
|
result.append((''.join(('py', version[0])), 'none', 'any'))
|
|
|
|
return set(result)
|
|
|
|
|
|
COMPATIBLE_TAGS = compatible_tags()
|
|
|
|
del compatible_tags
|
|
|
|
|
|
def is_compatible(wheel, tags=None):
|
|
if not isinstance(wheel, Wheel):
|
|
wheel = Wheel(wheel) # assume it's a filename
|
|
result = False
|
|
if tags is None:
|
|
tags = COMPATIBLE_TAGS
|
|
for ver, abi, arch in tags:
|
|
if ver in wheel.pyver and abi in wheel.abi and arch in wheel.arch:
|
|
result = True
|
|
break
|
|
return result
|