Add BST_REQUIRES_PREVIOUS_SOURCE_STAGE option
This is an element option that allows sources to be staged more
seperately where possible rather than on a per element option. When
there is a source that has this option set to true, all previously
listed sources will be cached with this, and sources listed afterwards
will be cached individually. If no sources have this option, all sources
will be cached seperately.
Fixes #892
diff --git a/buildstream/element.py b/buildstream/element.py
index 05884c0..e897992 100644
--- a/buildstream/element.py
+++ b/buildstream/element.py
@@ -107,6 +107,7 @@
from .storage.directory import Directory
from .storage._filebaseddirectory import FileBasedDirectory
+from .storage._casbaseddirectory import CasBasedDirectory
from .storage.directory import VirtualDirectoryError
@@ -1537,9 +1538,17 @@
if self.__sources:
- sourcecache = self._get_context().sourcecache
+ sourcecache = context.sourcecache
+ # find last required source
+ last_required_previous_ix = 0
+ for ix, source in enumerate(self.__sources):
+ if source.BST_REQUIRES_PREVIOUS_SOURCES_CACHE:
+ last_required_previous_ix = ix
+
try:
- import_dir = sourcecache.export(self.__sources[-1])
+ import_dir = CasBasedDirectory(context.get_cascache())
+ for source in self.__sources[last_required_previous_ix:]:
+ import_dir.import_files(sourcecache.export(source))
except SourceCacheError as e:
raise ElementError("Error trying to export source for {}: {}"
.format(self.name, e))
@@ -2192,24 +2201,32 @@
def _fetch(self, fetch_original=False):
previous_sources = []
sources = self.__sources
+ fetch_needed = False
if sources and not fetch_original:
- source = sources[-1]
- if self.__sourcecache.contains(source):
- return
+ last_requires_previous = 0
+ for ix, source in enumerate(self.__sources):
+ if source.BST_REQUIRES_PREVIOUS_SOURCES_CACHE:
+ last_requires_previous = ix
- # try and fetch from source cache
- if source._get_consistency() < Consistency.CACHED and \
- self.__sourcecache.has_fetch_remotes() and \
- not self.__sourcecache.contains(source):
- if self.__sourcecache.pull(source):
- return
+ for source in self.__sources[last_requires_previous:]:
+ if self.__sourcecache.contains(source):
+ continue
+
+ # try and fetch from source cache
+ if source._get_consistency() < Consistency.CACHED and \
+ self.__sourcecache.has_fetch_remotes():
+ if self.__sourcecache.pull(source):
+ continue
+
+ fetch_needed = True
# We need to fetch original sources
- for source in self.sources():
- source_consistency = source._get_consistency()
- if source_consistency != Consistency.CACHED:
- source._fetch(previous_sources)
- previous_sources.append(source)
+ if fetch_needed or fetch_original:
+ for source in self.sources():
+ source_consistency = source._get_consistency()
+ if source_consistency != Consistency.CACHED:
+ source._fetch(previous_sources)
+ previous_sources.append(source)
self.__cache_sources()
@@ -2264,12 +2281,27 @@
# Check if sources are cached, generating the source key if it hasn't been
def _source_cached(self):
if self.__sources:
- last_source = self.__sources[-1]
- if not last_source._key:
- last_source._generate_key(self.__sources[:-1])
- return self._get_context().sourcecache.contains(last_source)
- else:
- return True
+ last_requires_previous = 0
+ for ix, source in enumerate(self.__sources):
+ if source.BST_REQUIRES_PREVIOUS_SOURCES_CACHE:
+ last_requires_previous = ix
+
+ sourcecache = self._get_context().sourcecache
+ if not self.__sources[last_requires_previous]._key:
+ self.__sources[last_requires_previous]._generate_key(
+ self.__sources[:last_requires_previous])
+
+ # Go through individual sources
+ for source in self.__sources[last_requires_previous+1:]:
+ if not source._key:
+ source._generate_key([])
+ if not sourcecache.contains(source):
+ return False
+
+ if not sourcecache.contains(self.__sources[last_requires_previous]):
+ return False
+
+ return True
def _should_fetch(self, fetch_original=False):
""" return bool of if we need to run the fetch stage for this element
@@ -2918,7 +2950,18 @@
def __cache_sources(self):
sources = self.__sources
if sources and not self._source_cached():
- sources[-1]._cache(sources[:-1])
+ last_requires_previous = 0
+ for ix, source in enumerate(sources):
+ if source.BST_REQUIRES_PREVIOUS_SOURCES_CACHE:
+ last_requires_previous = ix
+
+ # cache last source that requires previous sources
+ self.__sourcecache.commit(sources[last_requires_previous],
+ sources[:last_requires_previous])
+
+ # commit all other sources by themselves
+ for source in sources[last_requires_previous+1:]:
+ self.__sourcecache.commit(source, [])
# __update_state_recursively()
#
diff --git a/buildstream/plugins/sources/patch.py b/buildstream/plugins/sources/patch.py
index 8e833b4..6da1ea0 100644
--- a/buildstream/plugins/sources/patch.py
+++ b/buildstream/plugins/sources/patch.py
@@ -52,6 +52,8 @@
class PatchSource(Source):
# pylint: disable=attribute-defined-outside-init
+ BST_REQUIRES_PREVIOUS_SOURCES_CACHE = True
+
def configure(self, node):
self.path = self.node_get_project_path(node, 'path',
check_is_file=True)
diff --git a/buildstream/source.py b/buildstream/source.py
index 6f4ff57..94cb82e 100644
--- a/buildstream/source.py
+++ b/buildstream/source.py
@@ -286,6 +286,15 @@
*Since: 1.4*
"""
+ BST_REQUIRES_PREVIOUS_SOURCES_CACHE = False
+ """Whether access to previous sources is required during cache
+
+ When set to True:
+ * all sources listed before current source in the given element will be
+ passed to the source when it's cached.
+ * This source can not be the first source for an element.
+ """
+
def __init__(self, context, project, meta, *, alias_override=None):
provenance = _yaml.node_get_provenance(meta.config)
super().__init__("{}-{}".format(meta.element_name, meta.element_index),
@@ -709,7 +718,10 @@
def _cache(self, previous_sources):
# stage the source into the source cache
- self.__source_cache.commit(self, previous_sources)
+ if self.BST_REQUIRES_PREVIOUS_SOURCES_CACHE:
+ self.__source_cache.commit(self, previous_sources)
+ else:
+ self.__source_cache.commit(self, [])
# Wrapper for stage() api which gives the source
# plugin a fully constructed path considering the
diff --git a/tests/sources/previous_source_access/plugins/sources/foo_transform.py b/tests/sources/previous_source_access/plugins/sources/foo_transform.py
index 8209464..bec4f99 100644
--- a/tests/sources/previous_source_access/plugins/sources/foo_transform.py
+++ b/tests/sources/previous_source_access/plugins/sources/foo_transform.py
@@ -18,6 +18,7 @@
# We need access to previous both at track time and fetch time
BST_REQUIRES_PREVIOUS_SOURCES_TRACK = True
BST_REQUIRES_PREVIOUS_SOURCES_FETCH = True
+ BST_REQUIRES_PREVIOUS_SOURCES_CACHE = True
@property
def mirror(self):
