| # Pylint doesn't play well with fixtures and dependency injection from pytest |
| # pylint: disable=redefined-outer-name |
| |
| import itertools |
| import os |
| |
| import pytest |
| |
| from buildstream._remote import RemoteSpec, RemoteType |
| from buildstream._artifactcache import ArtifactCache |
| from buildstream._project import Project |
| from buildstream.utils import _deduplicate |
| from buildstream import _yaml |
| from buildstream.exceptions import ErrorDomain, LoadErrorReason |
| |
| from buildstream.testing.runcli import cli # pylint: disable=unused-import |
| |
| from tests.testutils import dummy_context |
| |
| |
| DATA_DIR = os.path.dirname(os.path.realpath(__file__)) |
| cache1 = RemoteSpec(url="https://example.com/cache1", push=True) |
| cache2 = RemoteSpec(url="https://example.com/cache2", push=False) |
| cache3 = RemoteSpec(url="https://example.com/cache3", push=False) |
| cache4 = RemoteSpec(url="https://example.com/cache4", push=False) |
| cache5 = RemoteSpec(url="https://example.com/cache5", push=False) |
| cache6 = RemoteSpec(url="https://example.com/cache6", push=True, type=RemoteType.ALL) |
| cache7 = RemoteSpec(url="https://index.example.com/cache1", push=True, type=RemoteType.INDEX) |
| cache8 = RemoteSpec(url="https://storage.example.com/cache1", push=True, type=RemoteType.STORAGE) |
| |
| |
| # Generate cache configuration fragments for the user config and project config files. |
| # |
| def configure_remote_caches(override_caches, project_caches=None, user_caches=None): |
| type_strings = {RemoteType.INDEX: "index", RemoteType.STORAGE: "storage", RemoteType.ALL: "all"} |
| |
| if project_caches is None: |
| project_caches = [] |
| |
| if user_caches is None: |
| user_caches = [] |
| |
| user_config = {} |
| if len(user_caches) == 1: |
| user_config["artifacts"] = { |
| "url": user_caches[0].url, |
| "push": user_caches[0].push, |
| "type": type_strings[user_caches[0].type], |
| } |
| elif len(user_caches) > 1: |
| user_config["artifacts"] = [ |
| {"url": cache.url, "push": cache.push, "type": type_strings[cache.type]} for cache in user_caches |
| ] |
| |
| if len(override_caches) == 1: |
| user_config["projects"] = { |
| "test": { |
| "artifacts": { |
| "url": override_caches[0].url, |
| "push": override_caches[0].push, |
| "type": type_strings[override_caches[0].type], |
| } |
| } |
| } |
| elif len(override_caches) > 1: |
| user_config["projects"] = { |
| "test": { |
| "artifacts": [ |
| {"url": cache.url, "push": cache.push, "type": type_strings[cache.type]} |
| for cache in override_caches |
| ] |
| } |
| } |
| |
| project_config = {} |
| if project_caches: |
| if len(project_caches) == 1: |
| project_config.update( |
| { |
| "artifacts": { |
| "url": project_caches[0].url, |
| "push": project_caches[0].push, |
| "type": type_strings[project_caches[0].type], |
| } |
| } |
| ) |
| elif len(project_caches) > 1: |
| project_config.update( |
| { |
| "artifacts": [ |
| {"url": cache.url, "push": cache.push, "type": type_strings[cache.type]} |
| for cache in project_caches |
| ] |
| } |
| ) |
| |
| return user_config, project_config |
| |
| |
| # Test that parsing the remote artifact cache locations produces the |
| # expected results. |
| @pytest.mark.parametrize( |
| "override_caches, project_caches, user_caches", |
| [ |
| # The leftmost cache is the highest priority one in all cases here. |
| pytest.param([], [], [], id="empty-config"), |
| pytest.param([], [], [cache1, cache2], id="user-config"), |
| pytest.param([], [cache1, cache2], [cache3], id="project-config"), |
| pytest.param([cache1], [cache2], [cache3], id="project-override-in-user-config"), |
| pytest.param([cache1, cache2], [cache3, cache4], [cache5, cache6], id="list-order"), |
| pytest.param([cache1, cache2, cache1], [cache2], [cache2, cache1], id="duplicates"), |
| pytest.param([cache7, cache8], [], [cache1], id="split-caches"), |
| ], |
| ) |
| def test_artifact_cache_precedence(tmpdir, override_caches, project_caches, user_caches): |
| # Produce a fake user and project config with the cache configuration. |
| user_config, project_config = configure_remote_caches(override_caches, project_caches, user_caches) |
| project_config["name"] = "test" |
| project_config["min-version"] = "2.0" |
| |
| user_config_file = str(tmpdir.join("buildstream.conf")) |
| _yaml.roundtrip_dump(user_config, file=user_config_file) |
| |
| project_dir = tmpdir.mkdir("project") |
| project_config_file = str(project_dir.join("project.conf")) |
| _yaml.roundtrip_dump(project_config, file=project_config_file) |
| |
| with dummy_context(config=user_config_file) as context: |
| project = Project(str(project_dir), context) |
| project.ensure_fully_loaded() |
| |
| # Use the helper from the artifactcache module to parse our configuration. |
| parsed_cache_specs = ArtifactCache._configured_remote_cache_specs(context, project) |
| |
| # Verify that it was correctly read. |
| expected_cache_specs = list(_deduplicate(itertools.chain(override_caches, project_caches, user_caches))) |
| assert parsed_cache_specs == expected_cache_specs |
| |
| |
| # Assert that if either the client key or client cert is specified |
| # without specifying its counterpart, we get a comprehensive LoadError |
| # instead of an unhandled exception. |
| @pytest.mark.datafiles(DATA_DIR) |
| @pytest.mark.parametrize("config_key, config_value", [("client-cert", "client.crt"), ("client-key", "client.key")]) |
| def test_missing_certs(cli, datafiles, config_key, config_value): |
| project = os.path.join(datafiles.dirname, datafiles.basename, "missing-certs") |
| |
| project_conf = { |
| "name": "test", |
| "min-version": "2.0", |
| "artifacts": {"url": "https://cache.example.com:12345", "push": "true", config_key: config_value}, |
| } |
| project_conf_file = os.path.join(project, "project.conf") |
| _yaml.roundtrip_dump(project_conf, project_conf_file) |
| |
| # Use `pull` here to ensure we try to initialize the remotes, triggering the error |
| # |
| # This does not happen for a simple `bst show`. |
| result = cli.run(project=project, args=["artifact", "pull", "element.bst"]) |
| result.assert_main_error(ErrorDomain.LOAD, LoadErrorReason.INVALID_DATA) |
| |
| |
| # Assert that BuildStream complains when someone attempts to define |
| # only one type of storage. |
| @pytest.mark.datafiles(DATA_DIR) |
| @pytest.mark.parametrize( |
| "override_caches, project_caches, user_caches", |
| [ |
| # The leftmost cache is the highest priority one in all cases here. |
| pytest.param([], [], [cache7], id="index-user"), |
| pytest.param([], [], [cache8], id="storage-user"), |
| pytest.param([], [cache7], [], id="index-project"), |
| pytest.param([], [cache8], [], id="storage-project"), |
| pytest.param([cache7], [], [], id="index-override"), |
| pytest.param([cache8], [], [], id="storage-override"), |
| ], |
| ) |
| def test_only_one(cli, datafiles, override_caches, project_caches, user_caches): |
| project = os.path.join(datafiles.dirname, datafiles.basename, "only-one") |
| |
| # Produce a fake user and project config with the cache configuration. |
| user_config, project_config = configure_remote_caches(override_caches, project_caches, user_caches) |
| project_config["name"] = "test" |
| project_config["min-version"] = "2.0" |
| |
| cli.configure(user_config) |
| |
| project_config_file = os.path.join(project, "project.conf") |
| _yaml.roundtrip_dump(project_config, file=project_config_file) |
| |
| # Use `pull` here to ensure we try to initialize the remotes, triggering the error |
| # |
| # This does not happen for a simple `bst show`. |
| result = cli.run(project=project, args=["artifact", "pull", "element.bst"]) |
| result.assert_main_error(ErrorDomain.STREAM, None) |
| |
| |
| @pytest.mark.datafiles(DATA_DIR) |
| @pytest.mark.parametrize( |
| "artifacts_config", |
| ( |
| { |
| "url": "http://localhost.test", |
| "server-cert": "~/server.crt", |
| "client-cert": "~/client.crt", |
| "client-key": "~/client.key", |
| }, |
| [ |
| { |
| "url": "http://localhost.test", |
| "server-cert": "~/server.crt", |
| "client-cert": "~/client.crt", |
| "client-key": "~/client.key", |
| }, |
| { |
| "url": "http://localhost2.test", |
| "server-cert": "~/server2.crt", |
| "client-cert": "~/client2.crt", |
| "client-key": "~/client2.key", |
| }, |
| ], |
| ), |
| ) |
| @pytest.mark.parametrize("in_user_config", [True, False]) |
| def test_paths_for_artifact_config_are_expanded(tmpdir, monkeypatch, artifacts_config, in_user_config): |
| # Produce a fake user and project config with the cache configuration. |
| # user_config, project_config = configure_remote_caches(override_caches, project_caches, user_caches) |
| # project_config['name'] = 'test' |
| |
| monkeypatch.setenv("HOME", str(tmpdir.join("homedir"))) |
| |
| project_config = {"name": "test", "min-version": "2.0"} |
| user_config = {} |
| if in_user_config: |
| user_config["artifacts"] = artifacts_config |
| else: |
| project_config["artifacts"] = artifacts_config |
| |
| user_config_file = str(tmpdir.join("buildstream.conf")) |
| _yaml.roundtrip_dump(user_config, file=user_config_file) |
| |
| project_dir = tmpdir.mkdir("project") |
| project_config_file = str(project_dir.join("project.conf")) |
| _yaml.roundtrip_dump(project_config, file=project_config_file) |
| |
| with dummy_context(config=user_config_file) as context: |
| project = Project(str(project_dir), context) |
| project.ensure_fully_loaded() |
| |
| # Use the helper from the artifactcache module to parse our configuration. |
| parsed_cache_specs = ArtifactCache._configured_remote_cache_specs(context, project) |
| |
| if isinstance(artifacts_config, dict): |
| artifacts_config = [artifacts_config] |
| |
| # Build expected artifact config |
| artifacts_config = [ |
| RemoteSpec( |
| url=config["url"], |
| push=False, |
| server_cert=os.path.expanduser(config["server-cert"]), |
| client_cert=os.path.expanduser(config["client-cert"]), |
| client_key=os.path.expanduser(config["client-key"]), |
| ) |
| for config in artifacts_config |
| ] |
| |
| assert parsed_cache_specs == artifacts_config |