| # |
| # Copyright (C) 2016 Codethink Limited |
| # Copyright (C) 2018 Bloomberg Finance LP |
| # |
| # This program is free software; you can redistribute it and/or |
| # modify it under the terms of the GNU Lesser General Public |
| # License as published by the Free Software Foundation; either |
| # version 2 of the License, or (at your option) any later version. |
| # |
| # This library is distributed in the hope that it will be useful, |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| # Lesser General Public License for more details. |
| # |
| # You should have received a copy of the GNU Lesser General Public |
| # License along with this library. If not, see <http://www.gnu.org/licenses/>. |
| # |
| # Authors: |
| # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> |
| """ |
| BuildElement - Abstract class for build elements |
| ================================================ |
| The BuildElement class is a convenience element one can derive from for |
| implementing the most common case of element. |
| |
| |
| .. _core_buildelement_builtins: |
| |
| Built-in functionality |
| ---------------------- |
| The BuildElement base class provides built in functionality that could be |
| overridden by the individual plugins. |
| |
| This section will give a brief summary of how some of the common features work, |
| some of them or the variables they use will be further detailed in the following |
| sections. |
| |
| |
| The `strip-binaries` variable |
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| The `strip-binaries` variable is by default **empty**. You need to use the |
| appropiate commands depending of the system you are building. |
| If you are targetting Linux, ones known to work are the ones used by the |
| `freedesktop-sdk <https://freedesktop-sdk.io/>`_, you can take a look to them in their |
| `project.conf <https://gitlab.com/freedesktop-sdk/freedesktop-sdk/blob/freedesktop-sdk-18.08.21/project.conf#L74>`_ |
| |
| |
| Location for staging dependencies |
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| The BuildElement supports the "location" :term:`dependency configuration <Dependency configuration>`, |
| which means you can use this configuration for any BuildElement class. |
| |
| The "location" configuration defines where the dependency will be staged in the |
| build sandbox. |
| |
| **Example:** |
| |
| Here is an example of how one might stage some dependencies into |
| an alternative location while staging some elements in the sandbox root. |
| |
| .. code:: yaml |
| |
| # Stage these build dependencies in /opt |
| # |
| build-depends: |
| - baseproject.bst:opt-dependencies.bst |
| config: |
| location: /opt |
| |
| # Stage these tools in "/" and require them as |
| # runtime dependencies. |
| depends: |
| - baseproject.bst:base-tools.bst |
| |
| .. note:: |
| |
| The order of dependencies specified is not significant. |
| |
| The staging locations will be sorted so that elements are staged in parent |
| directories before subdirectories. |
| |
| |
| Location for running commands |
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| The ``command-subdir`` variable sets where the build commands will be executed, |
| if the directory does not exist it will be created, it is defined relative to |
| the buildroot. |
| |
| |
| Location for configuring the project |
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| The ``conf-root`` is defined by default as ``.`` and is the location that |
| specific build element can use to look for build configuration files. This is |
| used by elements such as autotools, cmake, distutils, meson, pip and qmake. |
| |
| The configuration commands are run in ``command-subdir`` and by default |
| ``conf-root`` is ``.`` so if ``conf-root`` is not set the configuration files |
| in ``command-subdir`` will be used. |
| |
| By setting ``conf-root`` to ``"%{build-root}/Source/conf_location"`` and your |
| source elements ``directory`` variable to ``Source`` then the configuration |
| files in the directory ``conf_location`` with in your Source will be used. |
| The current working directory when your configuration command is run will still |
| be wherever you set your ``command-subdir`` to be, regardless of where the |
| configure scripts are set with ``conf-root``. |
| |
| |
| Install Location |
| ~~~~~~~~~~~~~~~~ |
| You should not change the ``install-root`` variable as it is a special |
| writeable location in the sandbox but it is useful when writing custom |
| install instructions as it may need to be supplied as the ``DESTDIR``, please |
| see the :mod:`cmake <elements.cmake>` build element for example. |
| |
| |
| Abstract method implementations |
| ------------------------------- |
| |
| |
| Element.configure_sandbox() |
| ~~~~~~~~~~~~~~~~~~~~~~~~~~~ |
| In :func:`Element.configure_sandbox() <buildstream.element.Element.configure_sandbox>`, |
| the BuildElement will ensure that the sandbox locations described by the ``%{build-root}`` |
| and ``%{install-root}`` variables are marked and will be mounted read-write for the |
| :func:`assemble phase<buildstream.element.Element.configure_sandbox>`. |
| |
| The working directory for the sandbox will be configured to be the ``%{build-root}``, |
| unless the ``%{command-subdir}`` variable is specified for the element in question, |
| in which case the working directory will be configured as ``%{build-root}/%{command-subdir}``. |
| |
| |
| Element.stage() |
| ~~~~~~~~~~~~~~~ |
| In :func:`Element.stage() <buildstream.element.Element.stage>`, the BuildElement |
| will do the following operations: |
| |
| * Stage all of the build dependencies into the sandbox root. |
| |
| * Run the integration commands for all staged dependencies using |
| :func:`Element.integrate() <buildstream.element.Element.integrate>` |
| |
| * Stage any Source on the given element to the ``%{build-root}`` location |
| inside the sandbox, using |
| :func:`Element.stage_sources() <buildstream.element.Element.integrate>` |
| |
| |
| Element.prepare() |
| ~~~~~~~~~~~~~~~~~ |
| In :func:`Element.prepare() <buildstream.element.Element.prepare>`, |
| the BuildElement will run ``configure-commands``, which are used to |
| run one-off preparations that should not be repeated for a single |
| build directory. |
| |
| |
| Element.assemble() |
| ~~~~~~~~~~~~~~~~~~ |
| In :func:`Element.assemble() <buildstream.element.Element.assemble>`, the |
| BuildElement will proceed to run sandboxed commands which are expected to be |
| found in the element configuration. |
| |
| Commands are run in the following order: |
| |
| * ``build-commands``: Commands to build the element |
| * ``install-commands``: Commands to install the results into ``%{install-root}`` |
| * ``strip-commands``: Commands to strip debugging symbols installed binaries |
| |
| The result of the build is expected to end up in ``%{install-root}``, and |
| as such; Element.assemble() method will return the ``%{install-root}`` for |
| artifact collection purposes. |
| """ |
| |
| import os |
| |
| from .element import Element |
| from .sandbox import SandboxFlags |
| |
| |
| _command_steps = ["configure-commands", "build-commands", "install-commands", "strip-commands"] |
| |
| |
| class BuildElement(Element): |
| |
| ############################################################# |
| # Abstract Method Implementations # |
| ############################################################# |
| def configure(self, node): |
| |
| self.__commands = {} # pylint: disable=attribute-defined-outside-init |
| |
| # FIXME: Currently this forcefully validates configurations |
| # for all BuildElement subclasses so they are unable to |
| # extend the configuration |
| node.validate_keys(_command_steps) |
| |
| self._command_subdir = self.get_variable("command-subdir") # pylint: disable=attribute-defined-outside-init |
| |
| for command_name in _command_steps: |
| self.__commands[command_name] = node.get_str_list(command_name, []) |
| |
| def configure_dependencies(self, dependencies): |
| |
| self.__layout = {} # pylint: disable=attribute-defined-outside-init |
| |
| # FIXME: Currently this forcefully validates configurations |
| # for all BuildElement subclasses so they are unable to |
| # extend the configuration |
| |
| for dep in dependencies: |
| # Determine the location to stage each element, default is "/" |
| location = "/" |
| if dep.config: |
| dep.config.validate_keys(["location"]) |
| location = dep.config.get_str("location") |
| try: |
| element_list = self.__layout[location] |
| except KeyError: |
| element_list = [] |
| self.__layout[location] = element_list |
| |
| element_list.append((dep.element, dep.path)) |
| |
| def preflight(self): |
| pass |
| |
| def get_unique_key(self): |
| dictionary = {} |
| |
| for command_name, command_list in self.__commands.items(): |
| dictionary[command_name] = command_list |
| |
| if self._command_subdir: |
| dictionary["command-subdir"] = self._command_subdir |
| |
| # Specifying notparallel for a given element effects the |
| # cache key, while having the side effect of setting max-jobs to 1, |
| # which is normally automatically resolved and does not affect |
| # the cache key. |
| if self.get_variable("notparallel"): |
| dictionary["notparallel"] = True |
| |
| # Specify the layout in the key, if any of the elements are not going to |
| # be staged in "/" |
| # |
| if any(location for location in self.__layout if location != "/"): |
| sorted_locations = sorted(self.__layout) |
| layout_key = { |
| location: [dependency_path for _, dependency_path in self.__layout[location]] |
| for location in sorted_locations |
| } |
| dictionary["layout"] = layout_key |
| |
| return dictionary |
| |
| def configure_sandbox(self, sandbox): |
| build_root = self.get_variable("build-root") |
| install_root = self.get_variable("install-root") |
| |
| # Tell the sandbox to mount the build root and install root |
| sandbox.mark_directory(build_root) |
| sandbox.mark_directory(install_root) |
| |
| # Mark the artifact directories in the layout |
| for location in self.__layout: |
| sandbox.mark_directory(location, artifact=True) |
| |
| # Allow running all commands in a specified subdirectory |
| if self._command_subdir: |
| command_dir = os.path.join(build_root, self._command_subdir) |
| else: |
| command_dir = build_root |
| sandbox.set_work_directory(command_dir) |
| |
| # Setup environment |
| sandbox.set_environment(self.get_environment()) |
| |
| def stage(self, sandbox): |
| |
| # First stage it all |
| # |
| sorted_locations = sorted(self.__layout) |
| for location in sorted_locations: |
| with self.timed_activity("Staging dependencies at: {}".format(location), silent_nested=True): |
| element_list = [element for element, _ in self.__layout[location]] |
| self.stage_dependency_artifacts(sandbox, element_list, path=location) |
| |
| # Now integrate any elements staged in the root |
| # |
| root_list = self.__layout.get("/", None) |
| if root_list: |
| element_list = [element for element, _ in root_list] |
| with self.timed_activity("Integrating sandbox", silent_nested=True), sandbox.batch(SandboxFlags.NONE): |
| for dep in self.dependencies(element_list): |
| dep.integrate(sandbox) |
| |
| # Stage sources in the build root |
| self.stage_sources(sandbox, self.get_variable("build-root")) |
| |
| def assemble(self, sandbox): |
| # Run commands |
| for command_name in _command_steps: |
| commands = self.__commands[command_name] |
| if not commands or command_name == "configure-commands": |
| continue |
| |
| with sandbox.batch(SandboxFlags.ROOT_READ_ONLY, label="Running {}".format(command_name)): |
| for cmd in commands: |
| self.__run_command(sandbox, cmd) |
| |
| # %{install-root}/%{build-root} should normally not be written |
| # to - if an element later attempts to stage to a location |
| # that is not empty, we abort the build - in this case this |
| # will almost certainly happen. |
| staged_build = os.path.join(self.get_variable("install-root"), self.get_variable("build-root")) |
| |
| if os.path.isdir(staged_build) and os.listdir(staged_build): |
| self.warn( |
| "Writing to %{install-root}/%{build-root}.", |
| detail="Writing to this directory will almost " |
| + "certainly cause an error, since later elements " |
| + "will not be allowed to stage to %{build-root}.", |
| ) |
| |
| # Return the payload, this is configurable but is generally |
| # always the /buildstream-install directory |
| return self.get_variable("install-root") |
| |
| def prepare(self, sandbox): |
| commands = self.__commands["configure-commands"] |
| if not commands: |
| # No configure commands, nothing to do. |
| return |
| |
| # We need to ensure that the prepare() method is only called |
| # once in workspaces, because the changes will persist across |
| # incremental builds - not desirable, for example, in the case |
| # of autotools' `./configure`. |
| marker_filename = ".bst-prepared" |
| |
| if self._get_workspace(): |
| # We use an empty file as a marker whether prepare() has already |
| # been called in a previous build. |
| |
| vdir = sandbox.get_virtual_directory() |
| buildroot = self.get_variable("build-root") |
| buildroot_vdir = vdir.descend(*buildroot.lstrip(os.sep).split(os.sep)) |
| |
| if buildroot_vdir.exists(marker_filename): |
| # Already prepared |
| return |
| |
| with sandbox.batch(SandboxFlags.ROOT_READ_ONLY, label="Running configure-commands"): |
| for cmd in commands: |
| self.__run_command(sandbox, cmd) |
| |
| if self._get_workspace(): |
| sandbox._create_empty_file(marker_filename) |
| |
| def generate_script(self): |
| script = "" |
| for command_name in _command_steps: |
| commands = self.__commands[command_name] |
| |
| for cmd in commands: |
| script += "(set -ex; {}\n) || exit 1\n".format(cmd) |
| |
| return script |
| |
| ############################################################# |
| # Private Local Methods # |
| ############################################################# |
| def __run_command(self, sandbox, cmd): |
| # Note the -e switch to 'sh' means to exit with an error |
| # if any untested command fails. |
| # |
| sandbox.run(["sh", "-c", "-e", cmd + "\n"], SandboxFlags.ROOT_READ_ONLY, label=cmd) |
