buildstream/plugins/sources/pip.py - buildstream - Git at Google

 #
 #  Copyright 2018 Bloomberg Finance LP
 #
 #  This program is free software; you can redistribute it and/or
 #  modify it under the terms of the GNU Lesser General Public
 #  License as published by the Free Software Foundation; either
 #  version 2 of the License, or (at your option) any later version.
 #
 #  This library is distributed in the hope that it will be useful,
 #  but WITHOUT ANY WARRANTY; without even the implied warranty of
 #  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 #  Lesser General Public License for more details.
 #
 #  You should have received a copy of the GNU Lesser General Public
 #  License along with this library. If not, see <http://www.gnu.org/licenses/>.
 #
 #  Authors:
 #        Chandan Singh <csingh43@bloomberg.net>

 """
 pip - stage python packages using pip
 =====================================

 **Host depndencies:**

   * ``pip`` python module

 This plugin will download source distributions for specified packages using
 ``pip`` but will not install them. It is expected that the elements using this
 source will install the downloaded packages.

 Downloaded tarballs will be stored in a directory called ".bst_pip_downloads".

 **Usage:**

 .. code:: yaml

    # Specify the pip source kind
    kind: pip

    # Optionally specify index url, defaults to PyPi
    # This url is used to discover new versions of packages and download them
    # Projects intending to mirror their sources to a permanent location should
    # use an aliased url, and declare the alias in the project configuration
    url: https://mypypi.example.com/simple

    # Optionally specify the path to requirements files
    # Note that either 'requirements-files' or 'packages' must be defined
    requirements-files:
    - requirements.txt

    # Optionally specify a list of additional packages
    # Note that either 'requirements-files' or 'packages' must be defined
    packages:
    - flake8

    # Specify the ref. It is a list of strings of format
    # "<package-name>==<version>", separated by "\\n".
    # Usually this will be contents of a requirements.txt file where all
    # package versions have been frozen.
    ref: "flake8==3.5.0\\nmccabe==0.6.1\\npkg-resources==0.0.0\\npycodestyle==2.3.1\\npyflakes==1.6.0"

 See :ref:`built-in functionality doumentation <core_source_builtins>` for
 details on common configuration options for sources.

 .. note::

    The ``pip`` plugin is available since :ref:`format version 16 <project_format_version>`
 """

 import hashlib
 import os
 import re

 from buildstream import Consistency, Source, SourceError, utils

 _OUTPUT_DIRNAME = '.bst_pip_downloads'
 _PYPI_INDEX_URL = 'https://pypi.org/simple/'

 # Used only for finding pip command
 _PYTHON_VERSIONS = [
     'python',  # when running in a venv, we might not have the exact version
     'python2.7',
     'python3.0',
     'python3.1',
     'python3.2',
     'python3.3',
     'python3.4',
     'python3.5',
     'python3.6',
     'python3.7',
 ]

 # List of allowed extensions taken from
 # https://docs.python.org/3/distutils/sourcedist.html.
 # Names of source distribution archives must be of the form
 # '%{package-name}-%{version}.%{extension}'.
 _SDIST_RE = re.compile(
     r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$',
     re.IGNORECASE)


 class PipSource(Source):
     # pylint: disable=attribute-defined-outside-init

     # We need access to previous sources at track time to use requirements.txt
     # but not at fetch time as self.ref should contain sufficient information
     # for this plugin
     BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True

     def configure(self, node):
         self.node_validate(node, ['url', 'packages', 'ref', 'requirements-files'] +
                            Source.COMMON_CONFIG_KEYS)
         self.keyorder += ['url', 'packages', 'ref', 'requirements-files']
         self.ref = self.node_get_member(node, str, 'ref', None)
         self.original_url = self.node_get_member(node, str, 'url', _PYPI_INDEX_URL)
         self.index_url = self.translate_url(self.original_url)
         self.packages = self.node_get_member(node, list, 'packages', [])
         self.requirements_files = self.node_get_member(node, list, 'requirements-files', [])

         if not (self.packages or self.requirements_files):
             raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self))

     def preflight(self):
         # Try to find a pip version that supports download command
         self.host_pip = None
         for python in reversed(_PYTHON_VERSIONS):
             try:
                 host_python = utils.get_host_tool(python)
                 rc = self.call([host_python, '-m', 'pip', 'download', '--help'])
                 if rc == 0:
                     self.host_pip = [host_python, '-m', 'pip']
                     break
             except utils.ProgramNotFoundError:
                 pass

         if self.host_pip is None:
             raise SourceError("{}: Unable to find a suitable pip command".format(self))

     def get_unique_key(self):
         return [self.original_url, self.ref]

     def get_consistency(self):
         if not self.ref:
             return Consistency.INCONSISTENT
         if os.path.exists(self._mirror) and os.listdir(self._mirror):
             return Consistency.CACHED
         return Consistency.RESOLVED

     def get_ref(self):
         return self.ref

     def load_ref(self, node):
         self.ref = self.node_get_member(node, str, 'ref', None)

     def set_ref(self, ref, node):
         node['ref'] = self.ref = ref

     def track(self, previous_sources_dir):
         # XXX pip does not offer any public API other than the CLI tool so it
         # is not feasible to correctly parse the requirements file or to check
         # which package versions pip is going to install.
         # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
         # for details.
         # As a result, we have to wastefully install the packages during track.
         with self.tempdir() as tmpdir:
             install_args = self.host_pip + ['download',
                                             '--no-binary', ':all:',
                                             '--index-url', self.index_url,
                                             '--dest', tmpdir]
             for requirement_file in self.requirements_files:
                 fpath = os.path.join(previous_sources_dir, requirement_file)
                 install_args += ['-r', fpath]
             install_args += self.packages

             self.call(install_args, fail="Failed to install python packages")
             reqs = self._parse_sdist_names(tmpdir)

         return '\n'.join(["{}=={}".format(pkg, ver) for pkg, ver in reqs])

     def fetch(self):
         with self.tempdir() as tmpdir:
             packages = self.ref.strip().split('\n')
             package_dir = os.path.join(tmpdir, 'packages')
             os.makedirs(package_dir)
             self.call(self.host_pip + ['download',
                                        '--no-binary', ':all:',
                                        '--index-url', self.index_url,
                                        '--dest', package_dir] + packages,
                       fail="Failed to install python packages: {}".format(packages))

             # If the mirror directory already exists, assume that some other
             # process has fetched the sources before us and ensure that we do
             # not raise an error in that case.
             try:
                 utils.move_atomic(package_dir, self._mirror)
             except utils.DirectoryExistsError:
                 # Another process has beaten us and has fetched the sources
                 # before us.
                 pass
             except OSError as e:
                 raise SourceError("{}: Failed to move downloaded pip packages from '{}' to '{}': {}"
                                   .format(self, package_dir, self._mirror, e)) from e

     def stage(self, directory):
         with self.timed_activity("Staging Python packages", silent_nested=True):
             utils.copy_files(self._mirror, os.path.join(directory, _OUTPUT_DIRNAME))

     # Directory where this source should stage its files
     #
     @property
     def _mirror(self):
         if not self.ref:
             return None
         return os.path.join(self.get_mirror_directory(),
                             utils.url_directory_name(self.original_url),
                             hashlib.sha256(self.ref.encode()).hexdigest())

     # Parse names of downloaded source distributions
     #
     # Args:
     #    basedir (str): Directory containing source distribution archives
     #
     # Returns:
     #    (list): List of (package_name, version) tuples in sorted order
     #
     def _parse_sdist_names(self, basedir):
         reqs = []
         for f in os.listdir(basedir):
             pkg = _match_package_name(f)
             if pkg is not None:
                 reqs.append(pkg)

         return sorted(reqs)


 # Extract the package name and version of a source distribution
 #
 # Args:
 #    filename (str): Filename of the source distribution
 #
 # Returns:
 #    (tuple): A tuple of (package_name, version)
 #
 def _match_package_name(filename):
     pkg_match = _SDIST_RE.match(filename)
     if pkg_match is None:
         return None
     return pkg_match.groups()


 def setup():
     return PipSource
	#
	# Copyright 2018 Bloomberg Finance LP
	#
	# This program is free software; you can redistribute it and/or
	# modify it under the terms of the GNU Lesser General Public
	# License as published by the Free Software Foundation; either
	# version 2 of the License, or (at your option) any later version.
	#
	# This library is distributed in the hope that it will be useful,
	# but WITHOUT ANY WARRANTY; without even the implied warranty of
	# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
	# Lesser General Public License for more details.
	#
	# You should have received a copy of the GNU Lesser General Public
	# License along with this library. If not, see <http://www.gnu.org/licenses/>.
	#
	# Authors:
	# Chandan Singh <csingh43@bloomberg.net>

	"""
	pip - stage python packages using pip
	=====================================

	Host depndencies:

	* ``pip`` python module

	This plugin will download source distributions for specified packages using
	``pip`` but will not install them. It is expected that the elements using this
	source will install the downloaded packages.

	Downloaded tarballs will be stored in a directory called ".bst_pip_downloads".

	Usage:

	.. code:: yaml

	# Specify the pip source kind
	kind: pip

	# Optionally specify index url, defaults to PyPi
	# This url is used to discover new versions of packages and download them
	# Projects intending to mirror their sources to a permanent location should
	# use an aliased url, and declare the alias in the project configuration
	url: https://mypypi.example.com/simple

	# Optionally specify the path to requirements files
	# Note that either 'requirements-files' or 'packages' must be defined
	requirements-files:
	- requirements.txt

	# Optionally specify a list of additional packages
	# Note that either 'requirements-files' or 'packages' must be defined
	packages:
	- flake8

	# Specify the ref. It is a list of strings of format
	# "<package-name>==<version>", separated by "\\n".
	# Usually this will be contents of a requirements.txt file where all
	# package versions have been frozen.
	ref: "flake8==3.5.0\\nmccabe==0.6.1\\npkg-resources==0.0.0\\npycodestyle==2.3.1\\npyflakes==1.6.0"

	See :ref:`built-in functionality doumentation <core_source_builtins>` for
	details on common configuration options for sources.

	.. note::

	The ``pip`` plugin is available since :ref:`format version 16 <project_format_version>`
	"""

	import hashlib
	import os
	import re

	from buildstream import Consistency, Source, SourceError, utils

	_OUTPUT_DIRNAME = '.bst_pip_downloads'
	_PYPI_INDEX_URL = 'https://pypi.org/simple/'

	# Used only for finding pip command
	_PYTHON_VERSIONS = [
	'python', # when running in a venv, we might not have the exact version
	'python2.7',
	'python3.0',
	'python3.1',
	'python3.2',
	'python3.3',
	'python3.4',
	'python3.5',
	'python3.6',
	'python3.7',
	]

	# List of allowed extensions taken from
	# https://docs.python.org/3/distutils/sourcedist.html.
	# Names of source distribution archives must be of the form
	# '%{package-name}-%{version}.%{extension}'.
	_SDIST_RE = re.compile(
	r'^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar\|tar.bz2\|tar.gz\|tar.xz\|tar.Z\|zip)$',
	re.IGNORECASE)


	class PipSource(Source):
	# pylint: disable=attribute-defined-outside-init

	# We need access to previous sources at track time to use requirements.txt
	# but not at fetch time as self.ref should contain sufficient information
	# for this plugin
	BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True

	def configure(self, node):
	self.node_validate(node, ['url', 'packages', 'ref', 'requirements-files'] +
	Source.COMMON_CONFIG_KEYS)
	self.keyorder += ['url', 'packages', 'ref', 'requirements-files']
	self.ref = self.node_get_member(node, str, 'ref', None)
	self.original_url = self.node_get_member(node, str, 'url', _PYPI_INDEX_URL)
	self.index_url = self.translate_url(self.original_url)
	self.packages = self.node_get_member(node, list, 'packages', [])
	self.requirements_files = self.node_get_member(node, list, 'requirements-files', [])

	if not (self.packages or self.requirements_files):
	raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified". format(self))

	def preflight(self):
	# Try to find a pip version that supports download command
	self.host_pip = None
	for python in reversed(_PYTHON_VERSIONS):
	try:
	host_python = utils.get_host_tool(python)
	rc = self.call([host_python, '-m', 'pip', 'download', '--help'])
	if rc == 0:
	self.host_pip = [host_python, '-m', 'pip']
	break
	except utils.ProgramNotFoundError:
	pass

	if self.host_pip is None:
	raise SourceError("{}: Unable to find a suitable pip command".format(self))

	def get_unique_key(self):
	return [self.original_url, self.ref]

	def get_consistency(self):
	if not self.ref:
	return Consistency.INCONSISTENT
	if os.path.exists(self._mirror) and os.listdir(self._mirror):
	return Consistency.CACHED
	return Consistency.RESOLVED

	def get_ref(self):
	return self.ref

	def load_ref(self, node):
	self.ref = self.node_get_member(node, str, 'ref', None)

	def set_ref(self, ref, node):
	node['ref'] = self.ref = ref

	def track(self, previous_sources_dir):
	# XXX pip does not offer any public API other than the CLI tool so it
	# is not feasible to correctly parse the requirements file or to check
	# which package versions pip is going to install.
	# See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
	# for details.
	# As a result, we have to wastefully install the packages during track.
	with self.tempdir() as tmpdir:
	install_args = self.host_pip + ['download',
	'--no-binary', ':all:',
	'--index-url', self.index_url,
	'--dest', tmpdir]
	for requirement_file in self.requirements_files:
	fpath = os.path.join(previous_sources_dir, requirement_file)
	install_args += ['-r', fpath]
	install_args += self.packages

	self.call(install_args, fail="Failed to install python packages")
	reqs = self._parse_sdist_names(tmpdir)

	return '\n'.join(["{}=={}".format(pkg, ver) for pkg, ver in reqs])

	def fetch(self):
	with self.tempdir() as tmpdir:
	packages = self.ref.strip().split('\n')
	package_dir = os.path.join(tmpdir, 'packages')
	os.makedirs(package_dir)
	self.call(self.host_pip + ['download',
	'--no-binary', ':all:',
	'--index-url', self.index_url,
	'--dest', package_dir] + packages,
	fail="Failed to install python packages: {}".format(packages))

	# If the mirror directory already exists, assume that some other
	# process has fetched the sources before us and ensure that we do
	# not raise an error in that case.
	try:
	utils.move_atomic(package_dir, self._mirror)
	except utils.DirectoryExistsError:
	# Another process has beaten us and has fetched the sources
	# before us.
	pass
	except OSError as e:
	raise SourceError("{}: Failed to move downloaded pip packages from '{}' to '{}': {}"
	.format(self, package_dir, self._mirror, e)) from e

	def stage(self, directory):
	with self.timed_activity("Staging Python packages", silent_nested=True):
	utils.copy_files(self._mirror, os.path.join(directory, _OUTPUT_DIRNAME))

	# Directory where this source should stage its files
	#
	@property
	def _mirror(self):
	if not self.ref:
	return None
	return os.path.join(self.get_mirror_directory(),
	utils.url_directory_name(self.original_url),
	hashlib.sha256(self.ref.encode()).hexdigest())

	# Parse names of downloaded source distributions
	#
	# Args:
	# basedir (str): Directory containing source distribution archives
	#
	# Returns:
	# (list): List of (package_name, version) tuples in sorted order
	#
	def _parse_sdist_names(self, basedir):
	reqs = []
	for f in os.listdir(basedir):
	pkg = _match_package_name(f)
	if pkg is not None:
	reqs.append(pkg)

	return sorted(reqs)


	# Extract the package name and version of a source distribution
	#
	# Args:
	# filename (str): Filename of the source distribution
	#
	# Returns:
	# (tuple): A tuple of (package_name, version)
	#
	def _match_package_name(filename):
	pkg_match = _SDIST_RE.match(filename)
	if pkg_match is None:
	return None
	return pkg_match.groups()


	def setup():
	return PipSource