blob: d821f18e4f3cbb5ac67e5462d61301e57ea2e73b [file] [log] [blame]
import os
import re
import shutil
import itertools
import pytest
from tests.testutils import cli, create_repo, generate_junction
from buildstream import _yaml
from buildstream._exceptions import ErrorDomain
from . import configure_project
# Project directory
DATA_DIR = os.path.join(
os.path.dirname(os.path.realpath(__file__)),
"project",
)
def create_element(repo, name, path, dependencies, ref=None):
element = {
'kind': 'import',
'sources': [
repo.source_config(ref=ref)
],
'depends': dependencies
}
_yaml.dump(element, os.path.join(path, name))
@pytest.mark.datafiles(os.path.join(DATA_DIR))
@pytest.mark.parametrize("strict", [True, False], ids=["strict", "no-strict"])
@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
@pytest.mark.parametrize("track_targets,exceptions,tracked", [
# Test with no exceptions
(['0.bst'], [], ['0.bst', '2.bst', '3.bst', '4.bst', '5.bst', '6.bst', '7.bst']),
(['3.bst'], [], ['3.bst', '4.bst', '5.bst', '6.bst']),
(['2.bst', '3.bst'], [], ['2.bst', '3.bst', '4.bst', '5.bst', '6.bst', '7.bst']),
# Test excepting '2.bst'
(['0.bst'], ['2.bst'], ['0.bst', '3.bst', '4.bst', '5.bst', '6.bst']),
(['3.bst'], ['2.bst'], []),
(['2.bst', '3.bst'], ['2.bst'], ['3.bst', '4.bst', '5.bst', '6.bst']),
# Test excepting '2.bst' and '3.bst'
(['0.bst'], ['2.bst', '3.bst'], ['0.bst']),
(['3.bst'], ['2.bst', '3.bst'], []),
(['2.bst', '3.bst'], ['2.bst', '3.bst'], [])
])
def test_build_track(cli, datafiles, tmpdir, ref_storage, strict,
track_targets, exceptions, tracked):
project = os.path.join(datafiles.dirname, datafiles.basename)
dev_files_path = os.path.join(project, 'files', 'dev-files')
element_path = os.path.join(project, 'elements')
repo = create_repo('git', str(tmpdir))
ref = repo.create(dev_files_path)
configure_project(project, {
'ref-storage': ref_storage
})
cli.configure({
'projects': {
'test': {
'strict': strict
}
}
})
create_elements = {
'0.bst': [
'2.bst',
'3.bst'
],
'2.bst': [
'3.bst',
'7.bst'
],
'3.bst': [
'4.bst',
'5.bst',
'6.bst'
],
'4.bst': [],
'5.bst': [],
'6.bst': [
'5.bst'
],
'7.bst': []
}
initial_project_refs = {}
for element, dependencies in create_elements.items():
# Test the element inconsistency resolution by ensuring that
# only elements that aren't tracked have refs
if element in set(tracked):
# Elements which should not have a ref set
#
create_element(repo, element, element_path, dependencies)
elif ref_storage == 'project.refs':
# Store a ref in project.refs
#
create_element(repo, element, element_path, dependencies)
initial_project_refs[element] = [{'ref': ref}]
else:
# Store a ref in the element itself
#
create_element(repo, element, element_path, dependencies, ref=ref)
# Generate initial project.refs
if ref_storage == 'project.refs':
project_refs = {
'projects': {
'test': initial_project_refs
}
}
_yaml.dump(project_refs, os.path.join(project, 'project.refs'))
args = ['build']
args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets))
args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions))
args += ['0.bst']
result = cli.run(project=project, silent=True, args=args)
result.assert_success()
# Assert that the main target 0.bst is cached
assert cli.get_element_state(project, '0.bst') == 'cached'
# Assert that we tracked exactly the elements we expected to
tracked_elements = result.get_tracked_elements()
assert set(tracked_elements) == set(tracked)
# Delete element sources
source_dir = os.path.join(project, 'cache', 'sources')
shutil.rmtree(source_dir)
# Delete artifacts one by one and assert element states
for target in set(tracked):
cli.remove_artifact_from_cache(project, target)
# Assert that it's tracked
assert cli.get_element_state(project, target) == 'fetch needed'
# Assert there was a project.refs created, depending on the configuration
if ref_storage == 'project.refs':
assert os.path.exists(os.path.join(project, 'project.refs'))
else:
assert not os.path.exists(os.path.join(project, 'project.refs'))
# This tests a very specific scenario:
#
# o Local cache is empty
# o Strict mode is disabled
# o The build target has only build dependencies
# o The build is launched with --track-all
#
# In this scenario, we have encountered bugs where BuildStream returns
# successfully after tracking completes without ever pulling, fetching or
# building anything.
#
@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("strict", [True, False], ids=["strict", "no-strict"])
@pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
def test_build_track_all(cli, tmpdir, datafiles, strict, ref_storage):
project = os.path.join(datafiles.dirname, datafiles.basename)
subproject_path = os.path.join(project, 'files', 'sub-project')
subproject_element_path = os.path.join(project, 'files', 'sub-project', 'elements')
junction_path = os.path.join(project, 'elements', 'junction.bst')
element_path = os.path.join(project, 'elements')
dev_files_path = os.path.join(project, 'files', 'dev-files')
configure_project(project, {
'ref-storage': ref_storage
})
cli.configure({
'projects': {
'test': {
'strict': strict
}
}
})
# We need a repo for real trackable elements
repo = create_repo('git', str(tmpdir))
ref = repo.create(dev_files_path)
# Create a trackable element to depend on the cross junction element,
# this one has it's ref resolved already
create_element(repo, 'sub-target.bst', subproject_element_path, ['import-etc.bst'], ref=ref)
# Create a trackable element to depend on the cross junction element
create_element(repo, 'target.bst', element_path, [
{
'junction': 'junction.bst',
'filename': 'sub-target.bst'
}
])
# Create a repo to hold the subproject and generate a junction element for it
generate_junction(tmpdir, subproject_path, junction_path, store_ref=False)
# Now create a compose element at the top level
element = {
'kind': 'compose',
'depends': [
{
'filename': 'target.bst',
'type': 'build'
}
]
}
_yaml.dump(element, os.path.join(element_path, 'composed.bst'))
# Track the junction itself first.
result = cli.run(project=project, args=['track', 'junction.bst'])
result.assert_success()
# Build it with --track-all
result = cli.run(project=project, silent=True, args=['build', '--track-all', 'composed.bst'])
result.assert_success()
# Assert that the main target is cached as a result
assert cli.get_element_state(project, 'composed.bst') == 'cached'
@pytest.mark.datafiles(os.path.join(DATA_DIR))
@pytest.mark.parametrize("track_targets,exceptions,tracked", [
# Test with no exceptions
(['0.bst'], [], ['0.bst', '2.bst', '3.bst', '4.bst', '5.bst', '6.bst', '7.bst']),
(['3.bst'], [], ['3.bst', '4.bst', '5.bst', '6.bst']),
(['2.bst', '3.bst'], [], ['2.bst', '3.bst', '4.bst', '5.bst', '6.bst', '7.bst']),
# Test excepting '2.bst'
(['0.bst'], ['2.bst'], ['0.bst', '3.bst', '4.bst', '5.bst', '6.bst']),
(['3.bst'], ['2.bst'], []),
(['2.bst', '3.bst'], ['2.bst'], ['3.bst', '4.bst', '5.bst', '6.bst']),
# Test excepting '2.bst' and '3.bst'
(['0.bst'], ['2.bst', '3.bst'], ['0.bst']),
(['3.bst'], ['2.bst', '3.bst'], []),
(['2.bst', '3.bst'], ['2.bst', '3.bst'], [])
])
def test_build_track_update(cli, datafiles, tmpdir, track_targets,
exceptions, tracked):
project = os.path.join(datafiles.dirname, datafiles.basename)
dev_files_path = os.path.join(project, 'files', 'dev-files')
element_path = os.path.join(project, 'elements')
repo = create_repo('git', str(tmpdir))
ref = repo.create(dev_files_path)
create_elements = {
'0.bst': [
'2.bst',
'3.bst'
],
'2.bst': [
'3.bst',
'7.bst'
],
'3.bst': [
'4.bst',
'5.bst',
'6.bst'
],
'4.bst': [],
'5.bst': [],
'6.bst': [
'5.bst'
],
'7.bst': []
}
for element, dependencies in create_elements.items():
# We set a ref for all elements, so that we ensure that we
# only track exactly those elements that we want to track,
# even if others can be tracked
create_element(repo, element, element_path, dependencies, ref=ref)
repo.add_commit()
args = ['build']
args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets))
args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions))
args += ['0.bst']
result = cli.run(project=project, silent=True, args=args)
tracked_elements = result.get_tracked_elements()
assert set(tracked_elements) == set(tracked)
@pytest.mark.datafiles(os.path.join(DATA_DIR))
@pytest.mark.parametrize("track_targets,exceptions", [
# Test tracking the main target element, but excepting some of its
# children
(['0.bst'], ['6.bst']),
# Test only tracking a child element
(['3.bst'], []),
])
def test_build_track_inconsistent(cli, datafiles, tmpdir,
track_targets, exceptions):
project = os.path.join(datafiles.dirname, datafiles.basename)
dev_files_path = os.path.join(project, 'files', 'dev-files')
element_path = os.path.join(project, 'elements')
repo = create_repo('git', str(tmpdir))
repo.create(dev_files_path)
create_elements = {
'0.bst': [
'2.bst',
'3.bst'
],
'2.bst': [
'3.bst',
'7.bst'
],
'3.bst': [
'4.bst',
'5.bst',
'6.bst'
],
'4.bst': [],
'5.bst': [],
'6.bst': [
'5.bst'
],
'7.bst': []
}
for element, dependencies in create_elements.items():
# We don't add refs so that all elements *have* to be tracked
create_element(repo, element, element_path, dependencies)
args = ['build']
args += itertools.chain.from_iterable(zip(itertools.repeat('--track'), track_targets))
args += itertools.chain.from_iterable(zip(itertools.repeat('--track-except'), exceptions))
args += ['0.bst']
result = cli.run(project=project, args=args, silent=True)
result.assert_main_error(ErrorDomain.PIPELINE, "inconsistent-pipeline")
# Assert that if a build element has a dependency in the tracking
# queue it does not start building before tracking finishes.
@pytest.mark.datafiles(os.path.join(DATA_DIR))
@pytest.mark.parametrize("strict", ['--strict', '--no-strict'])
def test_build_track_track_first(cli, datafiles, tmpdir, strict):
project = os.path.join(datafiles.dirname, datafiles.basename)
dev_files_path = os.path.join(project, 'files', 'dev-files')
element_path = os.path.join(project, 'elements')
repo = create_repo('git', str(tmpdir))
ref = repo.create(dev_files_path)
create_elements = {
'0.bst': [
'1.bst'
],
'1.bst': [],
'2.bst': [
'0.bst'
]
}
for element, dependencies in create_elements.items():
# We set a ref so that 0.bst can already be built even if
# 1.bst has not been tracked yet.
create_element(repo, element, element_path, dependencies, ref=ref)
repo.add_commit()
# Build 1.bst and 2.bst first so we have an artifact for them
args = [strict, 'build', '2.bst']
result = cli.run(args=args, project=project, silent=True)
result.assert_success()
# Test building 0.bst while tracking 1.bst
cli.remove_artifact_from_cache(project, '0.bst')
args = [strict, 'build', '--track', '1.bst', '2.bst']
result = cli.run(args=args, project=project, silent=True)
result.assert_success()
# Assert that 1.bst successfully tracks before 0.bst builds
track_messages = re.finditer(r'\[track:1.bst\s*]', result.stderr)
build_0 = re.search(r'\[build:0.bst\s*] START', result.stderr).start()
assert all(track_message.start() < build_0 for track_message in track_messages)
# Assert that 2.bst is *only* rebuilt if we are in strict mode
build_2 = re.search(r'\[build:2.bst\s*] START', result.stderr)
if strict == '--strict':
assert build_2 is not None
else:
assert build_2 is None