pip.py: Remove the 'pip' source plugin, it's in bst-plugins-experimental
diff --git a/NEWS b/NEWS
index af362eb..19ad35e 100644
--- a/NEWS
+++ b/NEWS
@@ -24,6 +24,7 @@
variable was not included in the cache key correctly.
o The `pip` element has been removed. Please use the one from bst-plugins-experimental
+ o The `pip` source has been removed. Please use the one from bst-plugins-experimental
API
diff --git a/doc/source/core_plugins.rst b/doc/source/core_plugins.rst
index b407286..900c7e4 100644
--- a/doc/source/core_plugins.rst
+++ b/doc/source/core_plugins.rst
@@ -54,7 +54,6 @@
sources/git
sources/bzr
sources/patch
- sources/pip
.. _plugins_external:
diff --git a/src/buildstream/plugins/sources/pip.py b/src/buildstream/plugins/sources/pip.py
deleted file mode 100644
index c0885ce..0000000
--- a/src/buildstream/plugins/sources/pip.py
+++ /dev/null
@@ -1,263 +0,0 @@
-#
-# Copyright 2018 Bloomberg Finance LP
-#
-# This program is free software; you can redistribute it and/or
-# modify it under the terms of the GNU Lesser General Public
-# License as published by the Free Software Foundation; either
-# version 2 of the License, or (at your option) any later version.
-#
-# This library is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
-# Lesser General Public License for more details.
-#
-# You should have received a copy of the GNU Lesser General Public
-# License along with this library. If not, see <http://www.gnu.org/licenses/>.
-#
-# Authors:
-# Chandan Singh <csingh43@bloomberg.net>
-
-"""
-pip - stage python packages using pip
-=====================================
-
-**Host depndencies:**
-
- * ``pip`` python module
-
-This plugin will download source distributions for specified packages using
-``pip`` but will not install them. It is expected that the elements using this
-source will install the downloaded packages.
-
-Downloaded tarballs will be stored in a directory called ".bst_pip_downloads".
-
-**Usage:**
-
-.. code:: yaml
-
- # Specify the pip source kind
- kind: pip
-
- # Optionally specify index url, defaults to PyPi
- # This url is used to discover new versions of packages and download them
- # Projects intending to mirror their sources to a permanent location should
- # use an aliased url, and declare the alias in the project configuration
- url: https://mypypi.example.com/simple
-
- # Optionally specify the path to requirements files
- # Note that either 'requirements-files' or 'packages' must be defined
- requirements-files:
- - requirements.txt
-
- # Optionally specify a list of additional packages
- # Note that either 'requirements-files' or 'packages' must be defined
- packages:
- - flake8
-
- # Specify the ref. It is a list of strings of format
- # "<package-name>==<version>", separated by "\\n".
- # Usually this will be contents of a requirements.txt file where all
- # package versions have been frozen.
- ref: "flake8==3.5.0\\nmccabe==0.6.1\\npkg-resources==0.0.0\\npycodestyle==2.3.1\\npyflakes==1.6.0"
-
-See :ref:`built-in functionality doumentation <core_source_builtins>` for
-details on common configuration options for sources.
-"""
-
-import hashlib
-import os
-import re
-
-from buildstream import Source, SourceError, utils
-
-_OUTPUT_DIRNAME = ".bst_pip_downloads"
-_PYPI_INDEX_URL = "https://pypi.org/simple/"
-
-# Used only for finding pip command
-_PYTHON_VERSIONS = [
- "python", # when running in a venv, we might not have the exact version
- "python2.7",
- "python3.0",
- "python3.1",
- "python3.2",
- "python3.3",
- "python3.4",
- "python3.5",
- "python3.6",
- "python3.7",
- "python3.8",
-]
-
-# List of allowed extensions taken from
-# https://docs.python.org/3/distutils/sourcedist.html.
-# Names of source distribution archives must be of the form
-# '%{package-name}-%{version}.%{extension}'.
-_SDIST_RE = re.compile(r"^([\w.-]+?)-((?:[\d.]+){2,})\.(?:tar|tar.bz2|tar.gz|tar.xz|tar.Z|zip)$", re.IGNORECASE)
-
-
-class PipSource(Source):
- # pylint: disable=attribute-defined-outside-init
-
- BST_MIN_VERSION = "2.0"
-
- # We need access to previous sources at track time to use requirements.txt
- # but not at fetch time as self.ref should contain sufficient information
- # for this plugin
- BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True
-
- def configure(self, node):
- node.validate_keys(["url", "packages", "ref", "requirements-files"] + Source.COMMON_CONFIG_KEYS)
- self.ref = node.get_str("ref", None)
- self.original_url = node.get_str("url", _PYPI_INDEX_URL)
- self.index_url = self.translate_url(self.original_url)
- self.packages = node.get_str_list("packages", [])
- self.requirements_files = node.get_str_list("requirements-files", [])
-
- if not (self.packages or self.requirements_files):
- raise SourceError("{}: Either 'packages' or 'requirements-files' must be specified".format(self))
-
- def preflight(self):
- # Try to find a pip version that spports download command
- self.host_pip = None
- for python in reversed(_PYTHON_VERSIONS):
- try:
- host_python = utils.get_host_tool(python)
- rc = self.call([host_python, "-m", "pip", "download", "--help"])
- if rc == 0:
- self.host_pip = [host_python, "-m", "pip"]
- break
- except utils.ProgramNotFoundError:
- pass
-
- if self.host_pip is None:
- raise SourceError("{}: Unable to find a suitable pip command".format(self))
-
- def get_unique_key(self):
- return [self.original_url, self.ref]
-
- def is_cached(self):
- return os.path.exists(self._mirror) and os.listdir(self._mirror)
-
- def get_ref(self):
- return self.ref
-
- def load_ref(self, node):
- self.ref = node.get_str("ref", None)
-
- def set_ref(self, ref, node):
- node["ref"] = self.ref = ref
-
- def track(self, previous_sources_dir): # pylint: disable=arguments-differ
- # XXX pip does not offer any public API other than the CLI tool so it
- # is not feasible to correctly parse the requirements file or to check
- # which package versions pip is going to install.
- # See https://pip.pypa.io/en/stable/user_guide/#using-pip-from-your-program
- # for details.
- # As a result, we have to wastefully install the packages during track.
- with self.tempdir() as tmpdir:
- install_args = self.host_pip + [
- "download",
- "--no-binary",
- ":all:",
- "--index-url",
- self.index_url,
- "--dest",
- tmpdir,
- ]
- for requirement_file in self.requirements_files:
- fpath = os.path.join(previous_sources_dir, requirement_file)
- install_args += ["-r", fpath]
- install_args += self.packages
-
- self.call(install_args, fail="Failed to install python packages")
- reqs = self._parse_sdist_names(tmpdir)
-
- return "\n".join(["{}=={}".format(pkg, ver) for pkg, ver in reqs])
-
- def fetch(self): # pylint: disable=arguments-differ
- with self.tempdir() as tmpdir:
- packages = self.ref.strip().split("\n")
- package_dir = os.path.join(tmpdir, "packages")
- os.makedirs(package_dir)
- self.call(
- [
- *self.host_pip,
- "download",
- "--no-binary",
- ":all:",
- "--index-url",
- self.index_url,
- "--dest",
- package_dir,
- *packages,
- ],
- fail="Failed to install python packages: {}".format(packages),
- )
-
- # If the mirror directory already exists, assume that some other
- # process has fetched the sources before us and ensure that we do
- # not raise an error in that case.
- try:
- utils.move_atomic(package_dir, self._mirror)
- except utils.DirectoryExistsError:
- # Another process has beaten us and has fetched the sources
- # before us.
- pass
- except OSError as e:
- raise SourceError(
- "{}: Failed to move downloaded pip packages from '{}' to '{}': {}".format(
- self, package_dir, self._mirror, e
- )
- ) from e
-
- def stage(self, directory):
- with self.timed_activity("Staging Python packages", silent_nested=True):
- utils.copy_files(self._mirror, os.path.join(directory, _OUTPUT_DIRNAME))
-
- # Directory where this source should stage its files
- #
- @property
- def _mirror(self):
- if not self.ref:
- return None
- return os.path.join(
- self.get_mirror_directory(),
- utils.url_directory_name(self.original_url),
- hashlib.sha256(self.ref.encode()).hexdigest(),
- )
-
- # Parse names of downloaded source distributions
- #
- # Args:
- # basedir (str): Directory containing source distribution archives
- #
- # Returns:
- # (list): List of (package_name, version) tuples in sorted order
- #
- def _parse_sdist_names(self, basedir):
- reqs = []
- for f in os.listdir(basedir):
- pkg = _match_package_name(f)
- if pkg is not None:
- reqs.append(pkg)
-
- return sorted(reqs)
-
-
-# Extract the package name and version of a source distribution
-#
-# Args:
-# filename (str): Filename of the source distribution
-#
-# Returns:
-# (tuple): A tuple of (package_name, version)
-#
-def _match_package_name(filename):
- pkg_match = _SDIST_RE.match(filename)
- if pkg_match is None:
- return None
- return pkg_match.groups()
-
-
-def setup():
- return PipSource
diff --git a/tests/cachekey/project/sources/pip1.bst b/tests/cachekey/project/sources/pip1.bst
deleted file mode 100644
index ee69efa..0000000
--- a/tests/cachekey/project/sources/pip1.bst
+++ /dev/null
@@ -1,12 +0,0 @@
-kind: import
-
-sources:
-- kind: git
- url: https://example.com/foo/foobar.git
- ref: b99955530263172ed1beae52aed7a33885ef781f
-- kind: pip
- url: https://pypi.example.com/simple
- packages:
- - horses
- - ponies
- ref: 'horses==0.0.1\nponies==0.0.2'
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
deleted file mode 100644
index 844f4c5..0000000
--- a/tests/cachekey/project/sources/pip1.expected
+++ /dev/null
@@ -1 +0,0 @@
-70870ffa81e4527a2d812c809b1e99025c72d1ae289be38e45a5b22dc0262eac
\ No newline at end of file
diff --git a/tests/cachekey/project/target.bst b/tests/cachekey/project/target.bst
index cabf3f7..1c63317 100644
--- a/tests/cachekey/project/target.bst
+++ b/tests/cachekey/project/target.bst
@@ -13,7 +13,6 @@
- sources/patch1.bst
- sources/patch2.bst
- sources/patch3.bst
-- sources/pip1.bst
- sources/remote1.bst
- sources/remote2.bst
- sources/tar1.bst
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index cf38ddc..7f861c6 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-a57f84353d1528b3ecbd9993ac02a20d0355f2451e19700172c95575af9a5b48
\ No newline at end of file
+fbf0b4dfc7d28cc4b534ea1c0d8af38a72a51d540cf903bca2c04389aadbe135
diff --git a/tests/integration/pip_source.py b/tests/integration/pip_source.py
deleted file mode 100644
index 5d31497..0000000
--- a/tests/integration/pip_source.py
+++ /dev/null
@@ -1,177 +0,0 @@
-# Pylint doesn't play well with fixtures and dependency injection from pytest
-# pylint: disable=redefined-outer-name
-
-import os
-import pytest
-
-from buildstream import _yaml
-
-from buildstream.testing import cli_integration as cli # pylint: disable=unused-import
-from buildstream.testing.integration import assert_contains
-from buildstream.testing._utils.site import HAVE_SANDBOX
-
-from tests.testutils.python_repo import setup_pypi_repo # pylint: disable=unused-import
-
-
-pytestmark = pytest.mark.integration
-
-
-DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "project")
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_pip_source_import_packages(cli, datafiles, setup_pypi_repo):
- project = str(datafiles)
- checkout = os.path.join(cli.directory, "checkout")
- element_path = os.path.join(project, "elements")
- element_name = "pip/hello.bst"
-
- # check that exotically named packages are imported correctly
- myreqs_packages = "hellolib"
- dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"]
- mock_packages = {myreqs_packages: {package: {} for package in dependencies}}
-
- # create mock pypi repository
- pypi_repo = os.path.join(project, "files", "pypi-repo")
- os.makedirs(pypi_repo, exist_ok=True)
- setup_pypi_repo(mock_packages, pypi_repo)
-
- element = {
- "kind": "import",
- "sources": [
- {"kind": "local", "path": "files/pip-source"},
- {"kind": "pip", "url": "file://{}".format(os.path.realpath(pypi_repo)), "packages": [myreqs_packages]},
- ],
- }
- os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
- _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
-
- result = cli.run(project=project, args=["source", "track", element_name])
- assert result.exit_code == 0
-
- result = cli.run(project=project, args=["build", element_name])
- assert result.exit_code == 0
-
- result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
- assert result.exit_code == 0
-
- assert_contains(
- checkout,
- [
- "/.bst_pip_downloads",
- "/.bst_pip_downloads/hellolib-0.1.tar.gz",
- "/.bst_pip_downloads/app2-0.1.tar.gz",
- "/.bst_pip_downloads/app.3-0.1.tar.gz",
- "/.bst_pip_downloads/app-4-0.1.tar.gz",
- "/.bst_pip_downloads/app_5-0.1.tar.gz",
- "/.bst_pip_downloads/app.no.6-0.1.tar.gz",
- "/.bst_pip_downloads/app-no-7-0.1.tar.gz",
- "/.bst_pip_downloads/app_no_8-0.1.tar.gz",
- ],
- )
-
-
-@pytest.mark.datafiles(DATA_DIR)
-def test_pip_source_import_requirements_files(cli, datafiles, setup_pypi_repo):
- project = str(datafiles)
- checkout = os.path.join(cli.directory, "checkout")
- element_path = os.path.join(project, "elements")
- element_name = "pip/hello.bst"
-
- # check that exotically named packages are imported correctly
- myreqs_packages = "hellolib"
- dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"]
- mock_packages = {myreqs_packages: {package: {} for package in dependencies}}
-
- # create mock pypi repository
- pypi_repo = os.path.join(project, "files", "pypi-repo")
- os.makedirs(pypi_repo, exist_ok=True)
- setup_pypi_repo(mock_packages, pypi_repo)
-
- element = {
- "kind": "import",
- "sources": [
- {"kind": "local", "path": "files/pip-source"},
- {
- "kind": "pip",
- "url": "file://{}".format(os.path.realpath(pypi_repo)),
- "requirements-files": ["myreqs.txt"],
- },
- ],
- }
- os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
- _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
-
- result = cli.run(project=project, args=["source", "track", element_name])
- assert result.exit_code == 0
-
- result = cli.run(project=project, args=["build", element_name])
- assert result.exit_code == 0
-
- result = cli.run(project=project, args=["artifact", "checkout", element_name, "--directory", checkout])
- assert result.exit_code == 0
-
- assert_contains(
- checkout,
- [
- "/.bst_pip_downloads",
- "/.bst_pip_downloads/hellolib-0.1.tar.gz",
- "/.bst_pip_downloads/app2-0.1.tar.gz",
- "/.bst_pip_downloads/app.3-0.1.tar.gz",
- "/.bst_pip_downloads/app-4-0.1.tar.gz",
- "/.bst_pip_downloads/app_5-0.1.tar.gz",
- "/.bst_pip_downloads/app.no.6-0.1.tar.gz",
- "/.bst_pip_downloads/app-no-7-0.1.tar.gz",
- "/.bst_pip_downloads/app_no_8-0.1.tar.gz",
- ],
- )
-
-
-@pytest.mark.datafiles(DATA_DIR)
-@pytest.mark.skipif(not HAVE_SANDBOX, reason="Only available with a functioning sandbox")
-def test_pip_source_build(cli, datafiles, setup_pypi_repo):
- project = str(datafiles)
- element_path = os.path.join(project, "elements")
- element_name = "pip/hello.bst"
-
- # check that exotically named packages are imported correctly
- myreqs_packages = "hellolib"
- dependencies = ["app2", "app.3", "app-4", "app_5", "app.no.6", "app-no-7", "app_no_8"]
- mock_packages = {myreqs_packages: {package: {} for package in dependencies}}
-
- # create mock pypi repository
- pypi_repo = os.path.join(project, "files", "pypi-repo")
- os.makedirs(pypi_repo, exist_ok=True)
- setup_pypi_repo(mock_packages, pypi_repo)
-
- element = {
- "kind": "manual",
- "depends": ["base.bst"],
- "sources": [
- {"kind": "local", "path": "files/pip-source"},
- {
- "kind": "pip",
- "url": "file://{}".format(os.path.realpath(pypi_repo)),
- "requirements-files": ["myreqs.txt"],
- "packages": dependencies,
- },
- ],
- "config": {
- "install-commands": [
- "pip3 install --no-index --prefix %{install-root}/usr .bst_pip_downloads/*.tar.gz",
- "install app1.py %{install-root}/usr/bin/",
- ]
- },
- }
- os.makedirs(os.path.dirname(os.path.join(element_path, element_name)), exist_ok=True)
- _yaml.roundtrip_dump(element, os.path.join(element_path, element_name))
-
- result = cli.run(project=project, args=["source", "track", element_name])
- assert result.exit_code == 0
-
- result = cli.run(project=project, args=["build", element_name])
- assert result.exit_code == 0
-
- result = cli.run(project=project, args=["shell", element_name, "/usr/bin/app1.py"])
- assert result.exit_code == 0
- assert result.output == "Hello App1! This is hellolib\n"
diff --git a/tests/sources/pip.py b/tests/sources/pip.py
deleted file mode 100644
index 1eacb44..0000000
--- a/tests/sources/pip.py
+++ /dev/null
@@ -1,58 +0,0 @@
-# Pylint doesn't play well with fixtures and dependency injection from pytest
-# pylint: disable=redefined-outer-name
-
-import os
-import pytest
-
-from buildstream.exceptions import ErrorDomain
-from buildstream.plugins.sources.pip import _match_package_name
-from buildstream.testing import cli, generate_project # pylint: disable=unused-import
-
-DATA_DIR = os.path.join(os.path.dirname(os.path.realpath(__file__)), "pip",)
-
-
-# Test that without ref, consistency is set appropriately.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-ref"))
-def test_no_ref(cli, datafiles):
- project = str(datafiles)
- generate_project(project)
- assert cli.get_element_state(project, "target.bst") == "no reference"
-
-
-# Test that pip is not allowed to be the first source
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "first-source-pip"))
-def test_first_source(cli, datafiles):
- project = str(datafiles)
- generate_project(project)
- result = cli.run(project=project, args=["show", "target.bst"])
- result.assert_main_error(ErrorDomain.ELEMENT, None)
-
-
-# Test that error is raised when neither packges nor requirements files
-# have been specified
-@pytest.mark.datafiles(os.path.join(DATA_DIR, "no-packages"))
-def test_no_packages(cli, datafiles):
- project = str(datafiles)
- generate_project(project)
- result = cli.run(project=project, args=["show", "target.bst"])
- result.assert_main_error(ErrorDomain.SOURCE, None)
-
-
-# Test that pip source parses tar ball names correctly for the ref
-@pytest.mark.parametrize(
- "tarball, expected_name, expected_version",
- [
- ("dotted.package-0.9.8.tar.gz", "dotted.package", "0.9.8"),
- ("hyphenated-package-2.6.0.tar.gz", "hyphenated-package", "2.6.0"),
- ("underscore_pkg-3.1.0.tar.gz", "underscore_pkg", "3.1.0"),
- ("numbers2and5-1.0.1.tar.gz", "numbers2and5", "1.0.1"),
- ("multiple.dots.package-5.6.7.tar.gz", "multiple.dots.package", "5.6.7"),
- ("multiple-hyphens-package-1.2.3.tar.gz", "multiple-hyphens-package", "1.2.3"),
- ("multiple_underscore_pkg-3.4.5.tar.gz", "multiple_underscore_pkg", "3.4.5"),
- ("shortversion-1.0.tar.gz", "shortversion", "1.0"),
- ("longversion-1.2.3.4.tar.gz", "longversion", "1.2.3.4"),
- ],
-)
-def test_match_package_name(tarball, expected_name, expected_version):
- name, version = _match_package_name(tarball)
- assert (expected_name, expected_version) == (name, version)
diff --git a/tests/sources/pip/first-source-pip/target.bst b/tests/sources/pip/first-source-pip/target.bst
deleted file mode 100644
index e5f20ab..0000000
--- a/tests/sources/pip/first-source-pip/target.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: import
-description: pip should not be allowed to be the first source
-sources:
-- kind: pip
- packages:
- - flake8
diff --git a/tests/sources/pip/no-packages/file b/tests/sources/pip/no-packages/file
deleted file mode 100644
index 980a0d5..0000000
--- a/tests/sources/pip/no-packages/file
+++ /dev/null
@@ -1 +0,0 @@
-Hello World!
diff --git a/tests/sources/pip/no-packages/target.bst b/tests/sources/pip/no-packages/target.bst
deleted file mode 100644
index 0d8b948..0000000
--- a/tests/sources/pip/no-packages/target.bst
+++ /dev/null
@@ -1,6 +0,0 @@
-kind: import
-description: The kind of this element is irrelevant.
-sources:
-- kind: local
- path: file
-- kind: pip
diff --git a/tests/sources/pip/no-ref/file b/tests/sources/pip/no-ref/file
deleted file mode 100644
index 980a0d5..0000000
--- a/tests/sources/pip/no-ref/file
+++ /dev/null
@@ -1 +0,0 @@
-Hello World!
diff --git a/tests/sources/pip/no-ref/target.bst b/tests/sources/pip/no-ref/target.bst
deleted file mode 100644
index ec450b7..0000000
--- a/tests/sources/pip/no-ref/target.bst
+++ /dev/null
@@ -1,8 +0,0 @@
-kind: import
-description: The kind of this element is irrelevant.
-sources:
-- kind: local
- path: file
-- kind: pip
- packages:
- - flake8
