Download buildtrees on demand for bst shell --use-buildtree
Provide bst shell --use-buildtree the ability to attempt to
acquire missing buildtrees, given respective option, user
pull-buildtree context and remote availability.
_frontend/cli.py: Refactor logic for determining --use-buildtree
option with given opportunity to attempt pulling a non-local
buildtree. Element loaded with artifact_config to allow remote
querying.
_stream.py: With given user option and element state, construct
PullQueue to fetch remote buildtree. Continue or Error without
buildtree if cannot be attained.
tests/integration/build-tree.py: Update to support new usecases
diff --git a/buildstream/_frontend/cli.py b/buildstream/_frontend/cli.py
index ab190aa..34217ae 100644
--- a/buildstream/_frontend/cli.py
+++ b/buildstream/_frontend/cli.py
@@ -526,7 +526,7 @@
else:
scope = Scope.RUN
- use_buildtree = False
+ use_buildtree = None
with app.initialized():
if not element:
@@ -534,7 +534,8 @@
if not element:
raise AppError('Missing argument "ELEMENT".')
- dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE)
+ dependencies = app.stream.load_selection((element,), selection=PipelineSelection.NONE,
+ use_artifact_config=True)
element = dependencies[0]
prompt = app.shell_prompt(element)
mounts = [
@@ -543,20 +544,31 @@
]
cached = element._cached_buildtree()
- if cli_buildtree == "always":
- if cached:
- use_buildtree = True
- else:
- raise AppError("No buildtree is cached but the use buildtree option was specified")
- elif cli_buildtree == "never":
- pass
- elif cli_buildtree == "try":
- use_buildtree = cached
+ if cli_buildtree in ("always", "try"):
+ use_buildtree = cli_buildtree
+ if not cached and use_buildtree == "always":
+ click.echo("WARNING: buildtree is not cached locally, will attempt to pull from available remotes",
+ err=True)
else:
- if app.interactive and cached:
- use_buildtree = bool(click.confirm('Do you want to use the cached buildtree?'))
+ # If the value has defaulted to ask and in non interactive mode, don't consider the buildtree, this
+ # being the default behaviour of the command
+ if app.interactive and cli_buildtree == "ask":
+ if cached and bool(click.confirm('Do you want to use the cached buildtree?')):
+ use_buildtree = "always"
+ elif not cached:
+ try:
+ choice = click.prompt("Do you want to pull & use a cached buildtree?",
+ type=click.Choice(['try', 'always', 'never']),
+ err=True, show_choices=True)
+ except click.Abort:
+ click.echo('Aborting', err=True)
+ sys.exit(-1)
+
+ if choice != "never":
+ use_buildtree = choice
+
if use_buildtree and not element._cached_success():
- click.echo("Warning: using a buildtree from a failed build.")
+ click.echo("WARNING: using a buildtree from a failed build.", err=True)
try:
exitcode = app.stream.shell(element, scope, prompt,
diff --git a/buildstream/_stream.py b/buildstream/_stream.py
index afb0256..af736c9 100644
--- a/buildstream/_stream.py
+++ b/buildstream/_stream.py
@@ -115,7 +115,7 @@
elements, _ = self._load(targets, (),
selection=selection,
except_targets=except_targets,
- fetch_subprojects=False
+ fetch_subprojects=False,
use_artifact_config=use_artifact_config)
profile_end(Topics.LOAD_SELECTION, "_".join(t.replace(os.sep, '-') for t in targets))
@@ -134,7 +134,7 @@
# mounts (list of HostMount): Additional directories to mount into the sandbox
# isolate (bool): Whether to isolate the environment like we do in builds
# command (list): An argv to launch in the sandbox, or None
- # usebuildtree (bool): Wheather to use a buildtree as the source.
+ # usebuildtree (str): Whether to use a buildtree as the source, given cli option
#
# Returns:
# (int): The exit code of the launched shell
@@ -144,7 +144,7 @@
mounts=None,
isolate=False,
command=None,
- usebuildtree=False):
+ usebuildtree=None):
# Assert we have everything we need built, unless the directory is specified
# in which case we just blindly trust the directory, using the element
@@ -159,8 +159,31 @@
raise StreamError("Elements need to be built or downloaded before staging a shell environment",
detail="\n".join(missing_deps))
+ buildtree = False
+ # Check if we require a pull queue attempt, with given artifact state and context
+ if usebuildtree:
+ if not element._cached_buildtree():
+ require_buildtree = self._buildtree_pull_required([element])
+ # Attempt a pull queue for the given element if remote and context allow it
+ if require_buildtree:
+ self._message(MessageType.INFO, "Attempting to fetch missing artifact buildtree")
+ self._add_queue(PullQueue(self._scheduler))
+ self._enqueue_plan(require_buildtree)
+ self._run()
+ # Now check if the buildtree was successfully fetched
+ if element._cached_buildtree():
+ buildtree = True
+ if not buildtree:
+ if usebuildtree == "always":
+ raise StreamError("Buildtree is not cached locally or in available remotes")
+ else:
+ self._message(MessageType.INFO, """Buildtree is not cached locally or in available remotes,
+ shell will be loaded without it""")
+ else:
+ buildtree = True
+
return element._shell(scope, directory, mounts=mounts, isolate=isolate, prompt=prompt, command=command,
- usebuildtree=usebuildtree)
+ usebuildtree=buildtree)
# build()
#
diff --git a/tests/integration/build-tree.py b/tests/integration/build-tree.py
index b50d841..b1a41ae 100644
--- a/tests/integration/build-tree.py
+++ b/tests/integration/build-tree.py
@@ -101,7 +101,7 @@
'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
])
res.assert_success()
- assert "Warning: using a buildtree from a failed build" in res.output
+ assert "WARNING: using a buildtree from a failed build" in res.stderr
assert 'Hi' in res.output
@@ -141,7 +141,7 @@
res.assert_success()
-# This test checks for correct behaviour if a buildtree is not present.
+# This test checks for correct behaviour if a buildtree is not present in the local cache.
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.skipif(IS_LINUX and not HAVE_BWRAP, reason='Only available with bubblewrap on Linux')
def test_buildtree_options(cli, tmpdir, datafiles):
@@ -156,6 +156,7 @@
result = cli.run(project=project, args=['build', element_name])
result.assert_success()
assert cli.get_element_state(project, element_name) == 'cached'
+ assert share.has_artifact('test', element_name, cli.get_element_key(project, element_name))
# Discard the cache
cli.configure({
@@ -168,8 +169,6 @@
result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', element_name])
result.assert_success()
- # The above is the simplest way I know to create a local cache without any buildtrees.
-
# Check it's not using the cached build tree
res = cli.run(project=project, args=[
'shell', '--build', element_name, '--use-buildtree', 'never', '--', 'cat', 'test'
@@ -177,13 +176,6 @@
res.assert_shell_error()
assert 'Hi' not in res.output
- # Check it's not correctly handling the lack of buildtree
- res = cli.run(project=project, args=[
- 'shell', '--build', element_name, '--use-buildtree', 'try', '--', 'cat', 'test'
- ])
- res.assert_shell_error()
- assert 'Hi' not in res.output
-
# Check it's not using the cached build tree, default is to ask, and fall back to not
# for non interactive behavior
res = cli.run(project=project, args=[
@@ -192,9 +184,43 @@
res.assert_shell_error()
assert 'Hi' not in res.output
- # Check it's using the cached build tree
+ # Check correctly handling the lack of buildtree, with 'try' not attempting to
+ # pull the buildtree as the user context is by default set to not pull them
+ res = cli.run(project=project, args=[
+ 'shell', '--build', element_name, '--use-buildtree', 'try', '--', 'cat', 'test'
+ ])
+ assert 'Hi' not in res.output
+ assert 'Attempting to fetch missing artifact buildtrees' not in res.stderr
+ assert """Buildtree is not cached locally or in available remotes,
+ shell will be loaded without it"""
+
+ # Check correctly handling the lack of buildtree, with 'try' attempting and succeeding
+ # to pull the buildtree as the user context allow the pulling of buildtrees and it is
+ # available in the remote
+ res = cli.run(project=project, args=[
+ '--pull-buildtrees', 'shell', '--build', element_name, '--use-buildtree', 'try', '--', 'cat', 'test'
+ ])
+ assert 'Attempting to fetch missing artifact buildtree' in res.stderr
+ assert 'Hi' in res.output
+ shutil.rmtree(os.path.join(os.path.join(cli.directory, 'artifacts2')))
+ assert cli.get_element_state(project, element_name) != 'cached'
+
+ # Check it's not loading the shell at all with always set for the buildtree, when the
+ # user context does not allow for buildtree pulling
+ result = cli.run(project=project, args=['artifact', 'pull', '--deps', 'all', element_name])
+ result.assert_success()
res = cli.run(project=project, args=[
'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
])
res.assert_main_error(ErrorDomain.PROG_NOT_FOUND, None)
+ assert 'Buildtree is not cached locally or in available remotes' in res.stderr
assert 'Hi' not in res.output
+ assert 'Attempting to fetch missing artifact buildtree' not in res.stderr
+
+ # Check that when user context is set to pull buildtrees and a remote has the buildtree,
+ # 'always' will attempt and succeed at pulling the missing buildtree.
+ res = cli.run(project=project, args=[
+ '--pull-buildtrees', 'shell', '--build', element_name, '--use-buildtree', 'always', '--', 'cat', 'test'
+ ])
+ assert 'Hi' in res.output
+ assert 'Attempting to fetch missing artifact buildtree' in res.stderr
