| # |
| # Copyright (C) 2017 Codethink Limited |
| # |
| # This program is free software; you can redistribute it and/or |
| # modify it under the terms of the GNU Lesser General Public |
| # License as published by the Free Software Foundation; either |
| # version 2 of the License, or (at your option) any later version. |
| # |
| # This library is distributed in the hope that it will be useful, |
| # but WITHOUT ANY WARRANTY; without even the implied warranty of |
| # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
| # Lesser General Public License for more details. |
| # |
| # You should have received a copy of the GNU Lesser General Public |
| # License along with this library. If not, see <http://www.gnu.org/licenses/>. |
| # |
| # Authors: |
| # Tristan Van Berkom <tristan.vanberkom@codethink.co.uk> |
| |
| """ |
| compose - Compose the output of multiple elements |
| ================================================= |
| This element creates a selective composition of its dependencies. |
| |
| This is normally used at near the end of a pipeline to prepare |
| something for later deployment. |
| |
| Since this element's output includes its dependencies, it may only |
| depend on elements as `build` type dependencies. |
| |
| The default configuration and possible options are as such: |
| .. literalinclude:: ../../../buildstream/plugins/elements/compose.yaml |
| :language: yaml |
| """ |
| |
| import os |
| from buildstream import Element, Scope |
| |
| |
| # Element implementation for the 'compose' kind. |
| class ComposeElement(Element): |
| # pylint: disable=attribute-defined-outside-init |
| |
| # The compose element's output is its dependencies, so |
| # we must rebuild if the dependencies change even when |
| # not in strict build plans. |
| # |
| BST_STRICT_REBUILD = True |
| |
| # Compose artifacts must never have indirect dependencies, |
| # so runtime dependencies are forbidden. |
| BST_FORBID_RDEPENDS = True |
| |
| # This element ignores sources, so we should forbid them from being |
| # added, to reduce the potential for confusion |
| BST_FORBID_SOURCES = True |
| |
| # This plugin has been modified to avoid the use of Sandbox.get_directory |
| BST_VIRTUAL_DIRECTORY = True |
| |
| def configure(self, node): |
| self.node_validate(node, [ |
| 'integrate', 'include', 'exclude', 'include-orphans' |
| ]) |
| |
| # We name this variable 'integration' only to avoid |
| # collision with the Element.integrate() method. |
| self.integration = self.node_get_member(node, bool, 'integrate') |
| self.include = self.node_get_member(node, list, 'include') |
| self.exclude = self.node_get_member(node, list, 'exclude') |
| self.include_orphans = self.node_get_member(node, bool, 'include-orphans') |
| |
| self.keyorder += ['integrate', 'include', 'exclude', 'include-orphans'] |
| |
| def preflight(self): |
| pass |
| |
| def get_unique_key(self): |
| key = {'integrate': self.integration, |
| 'include': sorted(self.include), |
| 'orphans': self.include_orphans} |
| |
| if self.exclude: |
| key['exclude'] = sorted(self.exclude) |
| |
| return key |
| |
| def configure_sandbox(self, sandbox): |
| pass |
| |
| def stage(self, sandbox): |
| pass |
| |
| def assemble(self, sandbox): |
| |
| require_split = self.include or self.exclude or not self.include_orphans |
| |
| # Stage deps in the sandbox root |
| with self.timed_activity("Staging dependencies", silent_nested=True): |
| self.stage_dependency_artifacts(sandbox, Scope.BUILD) |
| |
| manifest = set() |
| if require_split: |
| with self.timed_activity("Computing split", silent_nested=True): |
| for dep in self.dependencies(Scope.BUILD): |
| files = dep.compute_manifest(include=self.include, |
| exclude=self.exclude, |
| orphans=self.include_orphans) |
| manifest.update(files) |
| |
| # Make a snapshot of all the files. |
| vbasedir = sandbox.get_virtual_directory() |
| modified_files = set() |
| removed_files = set() |
| added_files = set() |
| |
| # Run any integration commands provided by the dependencies |
| # once they are all staged and ready |
| if self.integration: |
| with self.timed_activity("Integrating sandbox"): |
| if require_split: |
| |
| # Make a snapshot of all the files before integration-commands are run. |
| snapshot = set(vbasedir.list_relative_paths()) |
| vbasedir.mark_unmodified() |
| |
| with sandbox.batch(0): |
| for dep in self.dependencies(Scope.BUILD): |
| dep.integrate(sandbox) |
| |
| if require_split: |
| # Calculate added, modified and removed files |
| post_integration_snapshot = vbasedir.list_relative_paths() |
| modified_files = set(vbasedir.list_modified_paths()) |
| basedir_contents = set(post_integration_snapshot) |
| for path in manifest: |
| if path in snapshot and path not in basedir_contents: |
| removed_files.add(path) |
| |
| for path in basedir_contents: |
| if path not in snapshot: |
| added_files.add(path) |
| self.info("Integration modified {}, added {} and removed {} files" |
| .format(len(modified_files), len(added_files), len(removed_files))) |
| |
| # The remainder of this is expensive, make an early exit if |
| # we're not being selective about what is to be included. |
| if not require_split: |
| return '/' |
| |
| # Do we want to force include files which were modified by |
| # the integration commands, even if they were not added ? |
| # |
| manifest.update(added_files) |
| manifest.difference_update(removed_files) |
| |
| # XXX We should be moving things outside of the build sandbox |
| # instead of into a subdir. The element assemble() method should |
| # support this in some way. |
| # |
| installdir = vbasedir.descend(['buildstream', 'install'], create=True) |
| |
| # We already saved the manifest for created files in the integration phase, |
| # now collect the rest of the manifest. |
| # |
| |
| lines = [] |
| if self.include: |
| lines.append("Including files from domains: " + ", ".join(self.include)) |
| else: |
| lines.append("Including files from all domains") |
| |
| if self.exclude: |
| lines.append("Excluding files from domains: " + ", ".join(self.exclude)) |
| |
| if self.include_orphans: |
| lines.append("Including orphaned files") |
| else: |
| lines.append("Excluding orphaned files") |
| |
| detail = "\n".join(lines) |
| |
| with self.timed_activity("Creating composition", detail=detail, silent_nested=True): |
| self.info("Composing {} files".format(len(manifest))) |
| installdir.import_files(vbasedir, files=manifest, can_link=True) |
| |
| # And we're done |
| return os.path.join(os.sep, 'buildstream', 'install') |
| |
| |
| # Plugin entry point |
| def setup(): |
| return ComposeElement |