_artifact.py: Store sandbox config and environment in artifacts
This commit adds sandbox config and environment variables to the
artifact, so that they can be used later to shell into a downloaded
artifact or to run commands such as `bst artifact checkout --integrate`
which require running sandboxes on downloaded artifacts.
Summary:
* _artifact.py:
- Store new data in the new proto digest, similar
to how we store public data.
- Added new accessors to extract these from a loaded artifact.
- Bump the proto version number for compatibility
* _versions.py: Bump the global artifact version number
* element.py: Call Artifact.cache() with new parameters
* tests/cachekey: Updated cache key test
diff --git a/src/buildstream/_artifact.py b/src/buildstream/_artifact.py
index c110e57..0b78a60 100644
--- a/src/buildstream/_artifact.py
+++ b/src/buildstream/_artifact.py
@@ -29,13 +29,15 @@
"""
import os
+from typing import Dict
from ._protos.buildstream.v2.artifact_pb2 import Artifact as ArtifactProto
from . import _yaml
from . import utils
+from .node import Node
from .types import _Scope
from .storage._casbaseddirectory import CasBasedDirectory
-
+from .sandbox._config import SandboxConfig
# An Artifact class to abstract artifact operations
# from the Element class
@@ -48,7 +50,7 @@
#
class Artifact:
- version = 0
+ version = 1
def __init__(self, element, context, *, strong_key=None, weak_key=None):
self._element = element
@@ -137,11 +139,13 @@
# sourcesvdir (Directory): Virtual Directoy object for the staged sources
# buildresult (tuple): bool, short desc and detailed desc of result
# publicdata (dict): dict of public data to commit to artifact metadata
+ # environment (dict): dict of the element's environment variables
+ # sandboxconfig (SandboxConfig): The element's SandboxConfig
#
# Returns:
# (int): The size of the newly cached artifact
#
- def cache(self, sandbox_build_dir, collectvdir, sourcesvdir, buildresult, publicdata):
+ def cache(self, sandbox_build_dir, collectvdir, sourcesvdir, buildresult, publicdata, environment, sandboxconfig):
context = self._context
element = self._element
@@ -180,6 +184,17 @@
artifact.public_data.CopyFrom(public_data_digest)
size += public_data_digest.size_bytes
+ # Store sandbox data, including SandboxConfig and environment variables
+ with utils._tempnamedfile_name(dir=self._tmpdir) as tmpname:
+ sandbox_config = sandboxconfig.to_dict()
+ sandbox_dict = {"environment": environment, "sandbox-config": sandbox_config}
+ sandboxdata = Node.from_dict(sandbox_dict)
+
+ _yaml.roundtrip_dump(sandboxdata, tmpname)
+ sandbox_data_digest = self._cas.add_object(path=tmpname, link_directly=True)
+ artifact.sandbox_data.CopyFrom(sandbox_data_digest)
+ size += sandbox_data_digest.size_bytes
+
# store build dependencies
for e in element._dependencies(_Scope.BUILD):
new_build = artifact.build_deps.add()
@@ -282,6 +297,46 @@
return data
+ # load_sandbox_config():
+ #
+ # Loads the sandbox configuration from the cached artifact
+ #
+ # Returns:
+ # The stored SandboxConfig object
+ #
+ def load_sandbox_config(self) -> SandboxConfig:
+
+ # Load the sandbox data from the artifact
+ artifact = self._get_proto()
+ meta_file = self._cas.objpath(artifact.sandbox_data)
+ data = _yaml.load(meta_file, shortname="sandbox.yaml")
+
+ # Extract the sandbox data
+ config = data.get_mapping("sandbox-config")
+
+ # Return a SandboxConfig
+ return SandboxConfig.new_from_node(config)
+
+ # load_environment():
+ #
+ # Loads the environment variables from the cached artifact
+ #
+ # Returns:
+ # The environment variables
+ #
+ def load_environment(self) -> Dict[str, str]:
+
+ # Load the sandbox data from the artifact
+ artifact = self._get_proto()
+ meta_file = self._cas.objpath(artifact.sandbox_data)
+ data = _yaml.load(meta_file, shortname="sandbox.yaml")
+
+ # Extract the environment
+ config = data.get_mapping("environment")
+
+ # Return the environment
+ return config.strip_node_info()
+
# load_build_result():
#
# Load the build result from the cached artifact
diff --git a/src/buildstream/_versions.py b/src/buildstream/_versions.py
index f97560b..37a7a0a 100644
--- a/src/buildstream/_versions.py
+++ b/src/buildstream/_versions.py
@@ -24,4 +24,4 @@
# or if buildstream was changed in a way which can cause
# the same cache key to produce something that is no longer
# the same.
-BST_CORE_ARTIFACT_VERSION = 9
+BST_CORE_ARTIFACT_VERSION = 10
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index 4a03e1c..1cc3c9b 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1849,7 +1849,15 @@
assert self.__artifact._cache_key is not None
with self.timed_activity("Caching artifact"):
- artifact_size = self.__artifact.cache(sandbox_build_dir, collectvdir, sourcesvdir, buildresult, publicdata)
+ artifact_size = self.__artifact.cache(
+ sandbox_build_dir,
+ collectvdir,
+ sourcesvdir,
+ buildresult,
+ publicdata,
+ self.__environment,
+ self.__sandbox_config,
+ )
if collect is not None and collectvdir is None:
raise ElementError(
diff --git a/tests/cachekey/project/elements/build1.expected b/tests/cachekey/project/elements/build1.expected
index 369036a..4213739 100644
--- a/tests/cachekey/project/elements/build1.expected
+++ b/tests/cachekey/project/elements/build1.expected
@@ -1 +1 @@
-e47d9b32ba894f1d454b66d8d30d3d20226b54c327e8b6a3893c9c55ff02378c
\ No newline at end of file
+5413d1e87db53d8c3120af005a86d221ac79d98a9253ddb49ab7b3d56133e698
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build2.expected b/tests/cachekey/project/elements/build2.expected
index 6e9dfd0..542c322 100644
--- a/tests/cachekey/project/elements/build2.expected
+++ b/tests/cachekey/project/elements/build2.expected
@@ -1 +1 @@
-ffe535e1ac311448584d9f0b17f51bb85c8cc9bd3c081a411f9c3ea464f5fcd7
\ No newline at end of file
+613857cdd1b6cb5e0f8fb98b1e32539803a787d03cbcc522c1fcf5b8db4fb260
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/build3.expected b/tests/cachekey/project/elements/build3.expected
index 7b3aabe..d914e5a 100644
--- a/tests/cachekey/project/elements/build3.expected
+++ b/tests/cachekey/project/elements/build3.expected
@@ -1 +1 @@
-c8bae677ec6af58543e4b779a295c9f6bee0c1eface38d9be62a66c247a68ba5
\ No newline at end of file
+e024f294c661f8df3a7b4463b51d17340c242ea90a352c5056199e2dc5fbd061
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index fd0ca75..4609f8b 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-3a033cd6b00a0b7e113f2f78f67e73923bdadad33e7d8825fabb0e75f0386417
\ No newline at end of file
+ca9e0e964d2277f8a8bc502962955d350932f9cf496c7eca7052719285352012
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index 5ab72fe..c5fda49 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-ec2cf079f662a497af5687d2458489fe3ad83810a2db1815ea489fb30f416fc9
\ No newline at end of file
+f293cd2989b3255d4bae77fcbf000163f299faf6303f2eba59b10a4f2a5fe060
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index 5a6620a..19a59af 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-d24fd1c4beca8b40aad24e8bd4a6106e1278e193b10872e1ed33b0829eb4831a
\ No newline at end of file
+2254279497818a92d7380ab00b30408914bb8d2b4adbef02ca125753ab1acadc
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index 2c4ab4c..b7e4962 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-37dd09028bd0a95e64b83df32a1446932f81a72c4c9a6a2cac5c0165ac61e29b
\ No newline at end of file
+1d842d5830bd62a931ee7478a1bf14ec6345a97cea8d94d78e239d04b00e97fd
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index 425c55e..c02a097 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-4f2eea4571545a5ce896b13320aa7e6c55ca874237fcb7ebeea017f8456e1b5c
\ No newline at end of file
+ba6db166dd116166426b11bf42f8319daa2152b430eb43d3df917fd710d676d7
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index d9e854e..7aa0afb 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-f27187b2e1b43a5d4d1855fc46fabe256d561d49d6aff2186a52d4d75315fe19
\ No newline at end of file
+99dd881c8b3435c8e724a3fa71f62fdd8e7b0a122c30f66e3a1b253b139b71b9
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index 263684b..f7448dd 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-815467a6401a32162cfbcff05f11ff366f9f2d11338bc89d9cf41715da7db89c
\ No newline at end of file
+969b77ee0c906eb3e7f027a4da7a2d68bf673cba38df681d5551882ea74c364c
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index 24d6a03..0b3e7fe 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-3fef8b7e07efe9b9599e32ce72b1f0f8ab31accd61788c94c57951770ab184c2
\ No newline at end of file
+bb438a86df411aefbe479d0495d84601014e230a14dfb4807bdb242a6fa20084
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index 5d9799f..b27de64 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-1220b2849910f60f864c2c325cda301809dcb9b730281862448508fbeae5fb91
\ No newline at end of file
+40b501ce478821f39823a6c02124b9b93bfe2b0a286bb80bbe61531357c02b24
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/bzr1.expected b/tests/cachekey/project/sources/bzr1.expected
index 4e66d6c..69c4179 100644
--- a/tests/cachekey/project/sources/bzr1.expected
+++ b/tests/cachekey/project/sources/bzr1.expected
@@ -1 +1 @@
-ce45db2c82a50eec39416dc857c7a676121a5276b396ea6c7917b123a932ea34
\ No newline at end of file
+2c6d7d9bbecc6b4ee47be936491c77bcbd9a31685fe2cb27cbb456ab01d84df0
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git1.expected b/tests/cachekey/project/sources/git1.expected
index 7c69745..81cb3dd 100644
--- a/tests/cachekey/project/sources/git1.expected
+++ b/tests/cachekey/project/sources/git1.expected
@@ -1 +1 @@
-f02ac8cc516819ee72f56bdc3c1a6e4bb3a154a506a361a8d34a63b37dfa8dc7
\ No newline at end of file
+ec9ffc62b2b41080bbfcaa58be70862235fd79bd2b86b76ded83dadb31b9067c
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git2.expected b/tests/cachekey/project/sources/git2.expected
index 4fa7db8..b80bc1c 100644
--- a/tests/cachekey/project/sources/git2.expected
+++ b/tests/cachekey/project/sources/git2.expected
@@ -1 +1 @@
-59964ee437e38e37dbbb78362118937e90defd89bdf358667394a35f6ed2ba46
\ No newline at end of file
+7836185b731178dd71a5516ea1c2e3ea415e51c9ccaf8b1a09eb4e328d44a5ef
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/git3.expected b/tests/cachekey/project/sources/git3.expected
index 91cb0f8..02470bf 100644
--- a/tests/cachekey/project/sources/git3.expected
+++ b/tests/cachekey/project/sources/git3.expected
@@ -1 +1 @@
-10dd7b3883cfe8fd8f7f29dd258098080227ee73c53ff825542d62d204f1e6c3
\ No newline at end of file
+bbd785fc38a9102f3eca0127f91a5106a204e813c272e687cebf4e27db2d0e55
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index 961785a..301a04d 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-13acaa66115bd1d374b236b0ddd2cef804df75b3aa28878f4200efa7c758b14c
\ No newline at end of file
+a5348ed332c979cf36d32c47305af704c4e68492994c0f05506de988fb1bac32
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index 762e7bc..0c46a41 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-214238ba50ead82896e27d95e833c3df990f9e3ab728fdb0e44a3b07f986ab3c
\ No newline at end of file
+c23efecb2fbfe5918e109a626c0b25395b575966700a5cf400b367ce47d6af3c
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch1.expected b/tests/cachekey/project/sources/patch1.expected
index 4593895..7dee823 100644
--- a/tests/cachekey/project/sources/patch1.expected
+++ b/tests/cachekey/project/sources/patch1.expected
@@ -1 +1 @@
-3b607c5657b600ab0e2c8b5ea09ed26fc65b3899b67d107676cf687b2cde374d
\ No newline at end of file
+0d02000882610679920ae10fb373aae949a05f78bb38067dd0bdd31f702c4ddc
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch2.expected b/tests/cachekey/project/sources/patch2.expected
index 221c9c7..7e685da 100644
--- a/tests/cachekey/project/sources/patch2.expected
+++ b/tests/cachekey/project/sources/patch2.expected
@@ -1 +1 @@
-1911adbe62cbd7d9f0ded708530274c7f591798bd0a0b5f7e8358f7fcda3066a
\ No newline at end of file
+df5efec58942441502a76434535bf968deefc1239d0e7115f5493156bf4842e0
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/patch3.expected b/tests/cachekey/project/sources/patch3.expected
index 1ad6d9f..93c8b13 100644
--- a/tests/cachekey/project/sources/patch3.expected
+++ b/tests/cachekey/project/sources/patch3.expected
@@ -1 +1 @@
-175ff52608a55a6af98466822e22f401c46b2659116ccfeee6dc78e09249da78
\ No newline at end of file
+f8a51d2fa8ca36f953841e02b64a95ddfda4b415aa74e4d84c20e217328ca3a6
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/pip1.expected b/tests/cachekey/project/sources/pip1.expected
index 59f1186..0b872f7 100644
--- a/tests/cachekey/project/sources/pip1.expected
+++ b/tests/cachekey/project/sources/pip1.expected
@@ -1 +1 @@
-20ac777cc56dfeafcf4543e0d9ed31108b32075c8a3e9d25d3deec16fbf7f246
\ No newline at end of file
+14a8d44049e574eeca422afab39aa8bf9514b7e8b1753c6bcd26317c0ca2e549
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index 9970589..8a087f0 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-4d13955e7eeb0f0d77b0a3a72e77acd8636bdc8284712975593742d0667f88f7
\ No newline at end of file
+fd13350d98b1aa5824cb6d3a2e562922e7df5d545993e016f2eba2ef253347d5
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index 139e4bf..92ef794 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-5f904fd43ca49f268af3141afe73116f1260800918a738afeaf51a94f99fffb2
\ No newline at end of file
+00021ba6cab57cac9cc18ad2ca68e390eba0c6e7e95e9caa64717d794989d74b
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 89a4920..66897ca 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-f516c30a7a493acd74e49c4ac91cf697dfd32e8821d4230becac100cdac7df64
\ No newline at end of file
+fa75a47c4b50ce5c87517614253a584d94a632209fa3e28b0473a0e17f355c6d
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index e9928d7..85a4ee0 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-eba29361b50cf14128dbf34362635bc2170474c8bb13c916638b2531174bf04a
\ No newline at end of file
+ca8e454d9be8c210c32d5b102a8694ed5b5098ef6f0a6daed8845aa0c471d310
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip1.expected b/tests/cachekey/project/sources/zip1.expected
index 41292f2..92296c0 100644
--- a/tests/cachekey/project/sources/zip1.expected
+++ b/tests/cachekey/project/sources/zip1.expected
@@ -1 +1 @@
-a759e15073fdeed9224e41af4c094464e268bb8ced95dedf6e6dc35ec524d003
\ No newline at end of file
+903d568d7c49092a3175c80fdccc385046ab6080bed8021798775fa4a6209186
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/zip2.expected b/tests/cachekey/project/sources/zip2.expected
index e723f48..c6f5ab7 100644
--- a/tests/cachekey/project/sources/zip2.expected
+++ b/tests/cachekey/project/sources/zip2.expected
@@ -1 +1 @@
-d9c5a347340a387c4cecf24c29ad4b3c524773e8224683380221a62ec820abd9
\ No newline at end of file
+7251cee8748e932cd55fa5c8c06aa6c6b35b1a8c555823c7c7eedbeb490e762a
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 9ea9991..495638e 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-cb3b1bbf3d8b7be1a0ee7305c1056e95225f84b2c2b5189cfee69a66dcbd0786
\ No newline at end of file
+af4ceb04c152f244e2f03a83d5103ceaed3d8c50097ba64c743db01aa4d32872
\ No newline at end of file
