tests/frontend/source_checkout.py

# Pylint doesn't play well with fixtures and dependency injection from pytest
# pylint: disable=redefined-outer-name

import os
import tarfile

import pytest

from buildstream.testing import cli  # pylint: disable=unused-import

from buildstream import utils, _yaml

# Project directory
DATA_DIR = os.path.join(
    os.path.dirname(os.path.realpath(__file__)),
    'project',
)


def generate_remote_import_element(input_path, output_path):
    return {
        'kind': 'import',
        'sources': [
            {
                'kind': 'remote',
                'url': 'file://{}'.format(input_path),
                'filename': output_path,
                'ref': utils.sha256sum(input_path),
            }
        ]
    }


@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize(
    "with_workspace,guess_element",
    [(True, True), (True, False), (False, False)],
    ids=["workspace-guess", "workspace-no-guess", "no-workspace-no-guess"]
)
def test_source_checkout(datafiles, cli, tmpdir_factory, with_workspace, guess_element):
    tmpdir = tmpdir_factory.mktemp("")
    project = str(datafiles)
    checkout = os.path.join(cli.directory, 'source-checkout')
    target = 'checkout-deps.bst'
    workspace = os.path.join(str(tmpdir), 'workspace')
    elm_cmd = [target] if not guess_element else []

    if with_workspace:
        ws_cmd = ['-C', workspace]
        result = cli.run(project=project, args=["workspace", "open", "--directory", workspace, target])
        result.assert_success()
    else:
        ws_cmd = []

    args = ws_cmd + ['source', 'checkout', '--deps', 'none', '--directory', checkout, *elm_cmd]
    result = cli.run(project=project, args=args)
    result.assert_success()

    assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))


@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize('force_flag', ['--force', '-f'])
def test_source_checkout_force(datafiles, cli, force_flag):
    project = str(datafiles)
    checkout = os.path.join(cli.directory, 'source-checkout')
    target = 'checkout-deps.bst'

    # Make the checkout directory with 'some-thing' inside it
    os.makedirs(os.path.join(checkout, 'some-thing'))

    result = cli.run(project=project, args=['source', 'checkout',
                                            force_flag,
                                            '--deps', 'none',
                                            '--directory', checkout,
                                            target])
    result.assert_success()

    assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))


@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_tar(datafiles, cli):
    project = str(datafiles)
    tar = os.path.join(cli.directory, 'source-checkout.tar')
    target = 'checkout-deps.bst'

    result = cli.run(project=project, args=['source', 'checkout',
                                            '--tar', tar,
                                            '--deps', 'none',
                                            target])
    result.assert_success()

    assert os.path.exists(tar)
    with tarfile.open(tar) as tf:
        expected_content = os.path.join(tar, 'checkout-deps', 'etc', 'buildstream', 'config')
        tar_members = [f.name for f in tf]
        for member in tar_members:
            assert member in expected_content


@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize("compression", [("gz"), ("xz"), ("bz2")])
def test_source_checkout_compressed_tar(datafiles, cli, compression):
    project = str(datafiles)
    tarfile_name = "source-checkout.tar" + compression
    tar = os.path.join(cli.directory, tarfile_name)
    target = 'checkout-deps.bst'

    result = cli.run(project=project, args=['source', 'checkout',
                                            '--tar', tar,
                                            '--compression', compression,
                                            '--deps', 'none',
                                            target])
    result.assert_success()
    tar = tarfile.open(name=tar, mode='r:' + compression)
    assert os.path.join('checkout-deps', 'etc', 'buildstream', 'config') in tar.getnames()


@pytest.mark.datafiles(DATA_DIR)
@pytest.mark.parametrize('deps', [('build'), ('none'), ('run'), ('all')])
def test_source_checkout_deps(datafiles, cli, deps):
    project = str(datafiles)
    checkout = os.path.join(cli.directory, 'source-checkout')
    target = 'checkout-deps.bst'

    result = cli.run(project=project, args=['source', 'checkout',
                                            '--directory', checkout,
                                            '--deps', deps,
                                            target])
    result.assert_success()

    # Sources of the target
    if deps == 'build':
        assert not os.path.exists(os.path.join(checkout, 'checkout-deps'))
    else:
        assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))

    # Sources of the target's build dependencies
    if deps in ('build', 'all'):
        assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))
    else:
        assert not os.path.exists(os.path.join(checkout, 'import-dev'))

    # Sources of the target's runtime dependencies
    if deps in ('run', 'all'):
        assert os.path.exists(os.path.join(checkout, 'import-bin', 'usr', 'bin', 'hello'))
    else:
        assert not os.path.exists(os.path.join(checkout, 'import-bin'))


@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_except(datafiles, cli):
    project = str(datafiles)
    checkout = os.path.join(cli.directory, 'source-checkout')
    target = 'checkout-deps.bst'

    result = cli.run(project=project, args=['source', 'checkout',
                                            '--directory', checkout,
                                            '--deps', 'all',
                                            '--except', 'import-bin.bst',
                                            target])
    result.assert_success()

    # Sources for the target should be present
    assert os.path.exists(os.path.join(checkout, 'checkout-deps', 'etc', 'buildstream', 'config'))

    # Sources for import-bin.bst should not be present
    assert not os.path.exists(os.path.join(checkout, 'import-bin'))

    # Sources for other dependencies should be present
    assert os.path.exists(os.path.join(checkout, 'import-dev', 'usr', 'include', 'pony.h'))


@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_fetch(datafiles, cli):
    project = str(datafiles)
    checkout = os.path.join(cli.directory, 'source-checkout')
    target = 'remote-import-dev.bst'
    target_path = os.path.join(project, 'elements', target)

    # Create an element with remote source
    element = generate_remote_import_element(
        os.path.join(project, 'files', 'dev-files', 'usr', 'include', 'pony.h'),
        'pony.h')
    _yaml.roundtrip_dump(element, target_path)

    # Testing implicit fetching requires that we do not have the sources
    # cached already
    assert cli.get_element_state(project, target) == 'fetch needed'

    args = ['source', 'checkout']
    args += [target, checkout]
    result = cli.run(project=project, args=['source', 'checkout', '--directory', checkout, target])

    result.assert_success()
    assert os.path.exists(os.path.join(checkout, 'remote-import-dev', 'pony.h'))


@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_build_scripts(cli, tmpdir, datafiles):
    project_path = str(datafiles)
    element_name = 'source-bundle/source-bundle-hello.bst'
    normal_name = 'source-bundle-source-bundle-hello'
    checkout = os.path.join(str(tmpdir), 'source-checkout')

    args = ['source', 'checkout', '--include-build-scripts', '--directory', checkout, element_name]
    result = cli.run(project=project_path, args=args)
    result.assert_success()

    # There sould be a script for each element (just one in this case) and a top level build script
    expected_scripts = ['build.sh', 'build-' + normal_name]
    for script in expected_scripts:
        assert script in os.listdir(checkout)


@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_tar_buildscripts(cli, tmpdir, datafiles):
    project_path = str(datafiles)
    element_name = 'source-bundle/source-bundle-hello.bst'
    normal_name = 'source-bundle-source-bundle-hello'
    tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')

    args = ['source', 'checkout', '--include-build-scripts', '--tar', tar_file, element_name]
    result = cli.run(project=project_path, args=args)
    result.assert_success()

    expected_scripts = ['build.sh', 'build-' + normal_name]

    with tarfile.open(tar_file, 'r') as tf:
        for script in expected_scripts:
            assert script in tf.getnames()


# Test that the --directory and --tar options conflict
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_options_tar_and_dir_conflict(cli, tmpdir, datafiles):
    project = str(datafiles)
    checkout = os.path.join(cli.directory, 'source-checkout')
    tar_file = os.path.join(str(tmpdir), 'source-checkout.tar')
    target = 'checkout-deps.bst'

    result = cli.run(project=project, args=['source', 'checkout',
                                            '--directory', checkout,
                                            '--tar', tar_file,
                                            target])

    assert result.exit_code != 0
    assert "ERROR: options --directory and --tar conflict" in result.stderr


# Test that the --compression option without --tar fails
@pytest.mark.datafiles(DATA_DIR)
def test_source_checkout_compression_without_tar(cli, tmpdir, datafiles):
    project = str(datafiles)
    checkout = os.path.join(cli.directory, 'source-checkout')
    target = 'checkout-deps.bst'

    result = cli.run(project=project, args=['source', 'checkout',
                                            '--directory', checkout,
                                            '--compression', 'xz',
                                            target])

    assert result.exit_code != 0
    assert "ERROR: --compression specified without --tar" in result.stderr