Merge pull request #1715 from apache/tristan/light-cache-keys-without-sandbox-run
Lightweight cache keys when not running the sandbox
diff --git a/src/buildstream/element.py b/src/buildstream/element.py
index a9c4e42..9382c8c 100644
--- a/src/buildstream/element.py
+++ b/src/buildstream/element.py
@@ -1669,11 +1669,12 @@
rootdir, output_file, output_file, self.__sandbox_config
) as sandbox: # noqa
+ # Ensure that the plugin does not run commands if it said that it wouldn't
+ #
+ # We only disable commands here in _assemble() instead of __sandbox() because
+ # the user might still run a shell on an element which itself does not run commands.
+ #
if not self.BST_RUN_COMMANDS:
- # Element doesn't need to run any commands in the sandbox.
- #
- # Disable Sandbox.run() to allow CasBasedDirectory for all
- # sandboxes.
sandbox._disable_run()
# By default, the dynamic public data is the same as the static public data.
@@ -2239,9 +2240,6 @@
# Generate dict that is used as base for all cache keys
if self.__cache_key_dict is None:
- # Filter out nocache variables from the element's environment
- cache_env = {key: value for key, value in self.__environment.items() if key not in self.__env_nocache}
-
project = self._get_project()
self.__cache_key_dict = {
@@ -2250,15 +2248,19 @@
"element-plugin-key": self.get_unique_key(),
"element-plugin-name": self.get_kind(),
"element-plugin-version": self.BST_ARTIFACT_VERSION,
- "sandbox": self.__sandbox_config.to_dict(),
- "environment": cache_env,
"public": self.__public.strip_node_info(),
}
self.__cache_key_dict["sources"] = self.__sources.get_unique_key()
-
self.__cache_key_dict["fatal-warnings"] = sorted(project._fatal_warnings)
+ # Calculate sandbox related factors if this element runs the sandbox at assemble time.
+ if self.BST_RUN_COMMANDS:
+ # Filter out nocache variables from the element's environment
+ cache_env = {key: value for key, value in self.__environment.items() if key not in self.__env_nocache}
+ self.__cache_key_dict["sandbox"] = self.__sandbox_config.to_dict()
+ self.__cache_key_dict["environment"] = cache_env
+
cache_key_dict = self.__cache_key_dict.copy()
cache_key_dict["dependencies"] = dependencies
if weak_cache_key is not None:
diff --git a/src/buildstream/plugins/elements/compose.py b/src/buildstream/plugins/elements/compose.py
index 90446c8..3884a4a 100644
--- a/src/buildstream/plugins/elements/compose.py
+++ b/src/buildstream/plugins/elements/compose.py
@@ -66,6 +66,10 @@
self.exclude = node.get_str_list("exclude")
self.include_orphans = node.get_bool("include-orphans")
+ # Inform the core that we will not need to run any commands in the sandbox
+ if not self.integration:
+ self.BST_RUN_COMMANDS = False
+
def preflight(self):
pass
diff --git a/src/buildstream/sandbox/sandbox.py b/src/buildstream/sandbox/sandbox.py
index 47d841f..8faa13b 100644
--- a/src/buildstream/sandbox/sandbox.py
+++ b/src/buildstream/sandbox/sandbox.py
@@ -64,6 +64,13 @@
self.collect = collect
+# An internal exception which can be used to explicitly trigger a bug / exception
+# which will be reported with a stack trace instead of reporting a user facing error
+#
+class _SandboxBug(Exception):
+ pass
+
+
class Sandbox:
"""Sandbox()
@@ -344,7 +351,7 @@
label: str = None
) -> Optional[int]:
if not self.__allow_run:
- raise SandboxError("Sandbox.run() has been disabled")
+ raise _SandboxBug("Element specified BST_RUN_COMMANDS as False but called Sandbox.run()")
# Fallback to the sandbox default settings for
# the cwd and env.
@@ -552,9 +559,11 @@
# _disable_run()
#
- # Raise exception if `Sandbox.run()` is called. This enables use of
- # CasBasedDirectory for faster staging when command execution is not
- # required.
+ # Raise exception if `Sandbox.run()` is called.
+ #
+ # This enforces an invariant by raising an exception if an element
+ # plugin ever set BST_RUN_COMMANDS to False but then proceeded to
+ # attempt to run the sandbox at assemble time.
#
def _disable_run(self):
self.__allow_run = False
diff --git a/tests/cachekey/project/elements/compose1.expected b/tests/cachekey/project/elements/compose1.expected
index 845cb64..1d4daa9 100644
--- a/tests/cachekey/project/elements/compose1.expected
+++ b/tests/cachekey/project/elements/compose1.expected
@@ -1 +1 @@
-7cca3e9df846040a0e1bd034d61e667ff5552f594377152cdb87ca7eb1ecf624
\ No newline at end of file
+8406283ca890663b317e055a1c4b8c18248b2c12388ab0900e6ee6ab12af8abd
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose2.expected b/tests/cachekey/project/elements/compose2.expected
index 1761103..1ad943c 100644
--- a/tests/cachekey/project/elements/compose2.expected
+++ b/tests/cachekey/project/elements/compose2.expected
@@ -1 +1 @@
-76d17f849d551f448fca92c23fb3e58d00f8b338507684d3e19ff2b7eed973ae
\ No newline at end of file
+616a126d43f56ecfa0613093ff3a2b545bfa6568cd03cb2ad8bae8595568aeab
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose3.expected b/tests/cachekey/project/elements/compose3.expected
index d7bb403..d19010c 100644
--- a/tests/cachekey/project/elements/compose3.expected
+++ b/tests/cachekey/project/elements/compose3.expected
@@ -1 +1 @@
-b69f69de6220a40105231f085db41b3ec6369a01cb5bb1087e3a92adbe883079
\ No newline at end of file
+f718a75e4fe35bfb2f2000897d0e20e120f3e23c96d6ae9221a29fdb32bb1334
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose4.expected b/tests/cachekey/project/elements/compose4.expected
index 34a64cc..d4cacc6 100644
--- a/tests/cachekey/project/elements/compose4.expected
+++ b/tests/cachekey/project/elements/compose4.expected
@@ -1 +1 @@
-738683bb7a8f9b879ba2ac447b94d7a05de6e8714e4d730117ebef85526c63de
\ No newline at end of file
+56a74b2d1bb065e9f83c0415290daca95017478fcbd13a5326e219a9e6a6dd01
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/compose5.expected b/tests/cachekey/project/elements/compose5.expected
index d1d8996..f639828 100644
--- a/tests/cachekey/project/elements/compose5.expected
+++ b/tests/cachekey/project/elements/compose5.expected
@@ -1 +1 @@
-3627e625bea80e5c66b20b3301138eacf942475336b53da180ed8036aee7cb46
\ No newline at end of file
+9ab2f1b18d7894ee3cf507c19ace3e46897089532004b60339fd2a63b06f3f59
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import1.expected b/tests/cachekey/project/elements/import1.expected
index 16d3fdf..25487fe 100644
--- a/tests/cachekey/project/elements/import1.expected
+++ b/tests/cachekey/project/elements/import1.expected
@@ -1 +1 @@
-abcca93a8c5d9f483e91e2abe97477ecc161d6cb9a2c4616852a6db05e8d3295
\ No newline at end of file
+9fa24d24cf3351f522694c0cf39440efabb4ec65d888284b7048285f3eab9429
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import2.expected b/tests/cachekey/project/elements/import2.expected
index bd955b3..db14ed8 100644
--- a/tests/cachekey/project/elements/import2.expected
+++ b/tests/cachekey/project/elements/import2.expected
@@ -1 +1 @@
-857ecf27a54c23b557e4ac4fe35f8e651a41de159a507393538de11b28aeb4bc
\ No newline at end of file
+bf23c7d9b5a3a68c6fb2ffbc617a596ae65726a17dbd406a4b10acf3247acadc
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/import3.expected b/tests/cachekey/project/elements/import3.expected
index 7124fdb..73a2090 100644
--- a/tests/cachekey/project/elements/import3.expected
+++ b/tests/cachekey/project/elements/import3.expected
@@ -1 +1 @@
-65d86ffc69220b5f9de7670794bdc2bb7f7979596b029030c319f82c6be22008
\ No newline at end of file
+982779f72790b19d7fdf59f5f871cb498e5989fec1ebf5f205128f04af016b25
\ No newline at end of file
diff --git a/tests/cachekey/project/elements/script1.expected b/tests/cachekey/project/elements/script1.expected
index b5a65b4..0f8a666 100644
--- a/tests/cachekey/project/elements/script1.expected
+++ b/tests/cachekey/project/elements/script1.expected
@@ -1 +1 @@
-9fe8b7edde3d883884a93f23673fc5e8271f1607ed8ba6ef0491057f2ec94cf4
\ No newline at end of file
+15bed1afee844f2b3ea2a433dcd93a15494ac09c0efc6108a2157013004d2748
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local1.expected b/tests/cachekey/project/sources/local1.expected
index 9ce54de..c9ebb9c 100644
--- a/tests/cachekey/project/sources/local1.expected
+++ b/tests/cachekey/project/sources/local1.expected
@@ -1 +1 @@
-526e79c6485baaaa037078e9ae916b0a27ea29571bc47733eff4ab344a98a994
\ No newline at end of file
+b1b40e934f8c51256dc37990a290c5c66e2e35e98ef497e48b8cf010d8ef712b
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/local2.expected b/tests/cachekey/project/sources/local2.expected
index 0280257..0c8016c 100644
--- a/tests/cachekey/project/sources/local2.expected
+++ b/tests/cachekey/project/sources/local2.expected
@@ -1 +1 @@
-e7647c5f651303969e6f554601293af511d3ec8ce1e845918555de3159353f6c
\ No newline at end of file
+8aff46847f26a99612355026d903b7dea1a18422cc1bf08017697501a01d08dc
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote1.expected b/tests/cachekey/project/sources/remote1.expected
index 38e03a5..31407cb 100644
--- a/tests/cachekey/project/sources/remote1.expected
+++ b/tests/cachekey/project/sources/remote1.expected
@@ -1 +1 @@
-d114cb0014edaa5ef0faf8bded09317cf66992aaad83a64464ba01ad71a49ec6
\ No newline at end of file
+42b9d1371f692ed06df503967ddda9c11518325f2d856ba33069e41cec4e10e4
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/remote2.expected b/tests/cachekey/project/sources/remote2.expected
index c3bdacf..17afde7 100644
--- a/tests/cachekey/project/sources/remote2.expected
+++ b/tests/cachekey/project/sources/remote2.expected
@@ -1 +1 @@
-17f4888bf8f25f99fb27ee8ce4e1608cf159ed59c71f55f3c60db40dfcb1d4a9
\ No newline at end of file
+1f226281aa215e3b2fbb6c301f663cc4bb2257e2c88352e7def71b2d90a2545f
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar1.expected b/tests/cachekey/project/sources/tar1.expected
index 858d74f..9955c70 100644
--- a/tests/cachekey/project/sources/tar1.expected
+++ b/tests/cachekey/project/sources/tar1.expected
@@ -1 +1 @@
-75120aeebb659b69a3b688976a05058ed73fa90ff3ff2b81b619aac4f2757469
\ No newline at end of file
+7813bf80ae77e66fa7736bd8a5c3eb121a797b2e2069ed5cd598db09e86ff84a
\ No newline at end of file
diff --git a/tests/cachekey/project/sources/tar2.expected b/tests/cachekey/project/sources/tar2.expected
index 3b12f8d..8f70ec7 100644
--- a/tests/cachekey/project/sources/tar2.expected
+++ b/tests/cachekey/project/sources/tar2.expected
@@ -1 +1 @@
-a9fa1deeb8ae46ccc9e35db4e04f4b724b2c2391cf663415bbc384ce3445948a
\ No newline at end of file
+16c0dace226ce67f3b08dad7cd51ed672fb084c7cb74e358771c226490c8cd35
\ No newline at end of file
diff --git a/tests/cachekey/project/target.expected b/tests/cachekey/project/target.expected
index 56301bf..617b828 100644
--- a/tests/cachekey/project/target.expected
+++ b/tests/cachekey/project/target.expected
@@ -1 +1 @@
-34a0ed236c80ba9069519d6f0cb280747b3c949bf6e877645d35b616eb7729b5
\ No newline at end of file
+c4f7317484ebf493139660bd002bd4d62e9fb8c305f7b76e6d814226e8abf37c
\ No newline at end of file
diff --git a/tests/frontend/completions.py b/tests/frontend/completions.py
index f734112..3155ebc 100644
--- a/tests/frontend/completions.py
+++ b/tests/frontend/completions.py
@@ -340,8 +340,8 @@
# Use hard coded artifact names, cache keys should be stable now
artifacts = [
- "test/import-bin/0b769809a4e12dd5060df8e8bb1cd960f6c93d1ed2a6a34350eac9272ea71e81",
- "test/import-bin/67844557f63f985ef38ba68d68e3dfdeda02f4b40a90f53707f0fc643245ccb8",
+ "test/import-bin/b8eabff4ad70f954d6ba751340cff8f8e85cddd1537904cd107889b73f7b7041",
+ "test/import-bin/c737117d716278363c8398879ab557446c6f35d3d7472b75cb2e956f622da704",
]
# Test autocompletion of the artifact
diff --git a/tox.ini b/tox.ini
index cf21f8e..33462b7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -75,6 +75,10 @@
py{37,38,39,310}: BST_TEST_XDG_CACHE_HOME = {envtmpdir}/cache
py{37,38,39,310}: BST_TEST_XDG_CONFIG_HOME = {envtmpdir}/config
py{37,38,39,310}: BST_TEST_XDG_DATA_HOME = {envtmpdir}/share
+
+ # This is required to run tests with python 3.7
+ py37: SETUPTOOLS_ENABLE_FEATURES = "legacy-editable"
+
# This is required to get coverage for Cython
py{37,38,39,310}-!nocover: BST_CYTHON_TRACE = 1
randomized: PYTEST_ADDOPTS="--random-order-bucket=global"
