| #!/usr/bin/env python3 |
| # Licensed to the Apache Software Foundation (ASF) under one |
| # or more contributor license agreements. See the NOTICE file |
| # distributed with this work for additional information |
| # regarding copyright ownership. The ASF licenses this file |
| # to you under the Apache License, Version 2.0 (the |
| # "License"); you may not use this file except in compliance |
| # with the License. You may obtain a copy of the License at |
| # |
| # http://www.apache.org/licenses/LICENSE-2.0 |
| # |
| # Unless required by applicable law or agreed to in writing, |
| # software distributed under the License is distributed on an |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| # KIND, either express or implied. See the License for the |
| # specific language governing permissions and limitations |
| # under the License. |
| |
| import json |
| import os |
| import re |
| import sys |
| from dataclasses import dataclass, field |
| from pathlib import Path |
| from typing import Any |
| |
| # The path for current script. |
| SCRIPT_PATH = Path(__file__).parent.absolute() |
| # The path for `.github` dir. |
| GITHUB_DIR = SCRIPT_PATH.parent.parent |
| # The project dir for opendal. |
| PROJECT_DIR = GITHUB_DIR.parent |
| |
| LANGUAGE_BINDING = ["java", "python", "nodejs", "go", "c", "cpp"] |
| |
| BIN = [] |
| |
| INTEGRATIONS = ["object_store"] |
| |
| |
| def provided_cases() -> list[dict[str, str]]: |
| root_dir = f"{GITHUB_DIR}/services" |
| |
| cases = [ |
| { |
| "service": service, |
| "setup": setup, |
| "feature": "services-{}".format(service.replace("_", "-")), |
| "content": Path( |
| os.path.join(root_dir, service, setup, "action.yml") |
| ).read_text(), |
| } |
| for service in os.listdir(root_dir) |
| for setup in os.listdir(os.path.join(root_dir, service)) |
| if os.path.exists(os.path.join(root_dir, service, setup, "action.yml")) |
| ] |
| |
| # Check if this workflow needs to read secrets. |
| # |
| # We will check if pattern `op://services` exist in content. |
| if not os.getenv("GITHUB_HAS_SECRETS") == "true": |
| cases[:] = [v for v in cases if "op://services" not in v["content"]] |
| |
| # Remove content from cases. |
| cases = [ |
| { |
| "setup": v["setup"], |
| "service": v["service"], |
| "feature": v["feature"], |
| } |
| for v in cases |
| ] |
| |
| # Make sure the order is stable. |
| sorted_cases = sorted(cases, key=lambda x: (x["service"], x["setup"])) |
| return sorted_cases |
| |
| |
| @dataclass |
| class Hint: |
| # Is core affected? |
| core: bool = field(default=False, init=False) |
| # Is binding java affected? |
| binding_java: bool = field(default=False, init=False) |
| # Is binding python affected? |
| binding_python: bool = field(default=False, init=False) |
| # Is binding nodejs affected? |
| binding_nodejs: bool = field(default=False, init=False) |
| # Is binding go affected? |
| binding_go: bool = field(default=False, init=False) |
| # Is binding c affected? |
| binding_c: bool = field(default=False, init=False) |
| # Is binding cpp affected? |
| binding_cpp: bool = field(default=False, init=False) |
| # Is bin ofs affected? |
| bin_ofs: bool = field(default=False, init=False) |
| # Is integration object_store affected ? |
| integration_object_store: bool = field(default=False, init=False) |
| |
| # Should we run all services tests? |
| all_service: bool = field(default=False, init=False) |
| # affected services set. |
| services: set = field(default_factory=set, init=False) |
| |
| |
| def calculate_hint(changed_files: list[str]) -> Hint: |
| hint = Hint() |
| |
| # Remove all files that end with `.md` |
| changed_files = [f for f in changed_files if not f.endswith(".md")] |
| |
| for p in changed_files: |
| # workflow behavior tests affected |
| if p == ".github/workflows/test_behavior.yml": |
| hint.core = True |
| for language in LANGUAGE_BINDING: |
| setattr(hint, f"binding_{language}", True) |
| for integration in INTEGRATIONS: |
| setattr(hint, f"integration_{integration}", True) |
| hint.all_service = True |
| |
| if p == ".github/workflows/test_behavior_core.yml": |
| hint.core = True |
| hint.all_service = True |
| |
| for language in LANGUAGE_BINDING: |
| if p == f".github/workflows/test_behavior_binding_{language}.yml": |
| setattr(hint, f"binding_{language}", True) |
| hint.all_service = True |
| |
| for bin in BIN: |
| if p == f".github/workflows/test_behavior_bin_{bin}.yml": |
| setattr(hint, f"bin_{bin}", True) |
| hint.all_service = True |
| |
| for integration in INTEGRATIONS: |
| if p == f".github/workflows/test_behavior_integration_{integration}.yml": |
| setattr(hint, f"integration_{integration}", True) |
| hint.all_service = True |
| |
| # core affected |
| if ( |
| p.startswith("core/") |
| and not p.startswith("core/benches/") |
| and not p.startswith("core/edge/") |
| and not p.startswith("core/fuzz/") |
| and not p.startswith("core/src/services/") |
| and not p.startswith("core/src/docs/") |
| ): |
| hint.core = True |
| hint.binding_java = True |
| hint.binding_python = True |
| hint.binding_nodejs = True |
| hint.binding_go = True |
| hint.binding_c = True |
| hint.binding_cpp = True |
| hint.bin_ofs = True |
| for integration in INTEGRATIONS: |
| setattr(hint, f"integration_{integration}", True) |
| hint.all_service = True |
| |
| # language binding affected |
| for language in LANGUAGE_BINDING: |
| if p.startswith(f"bindings/{language}/"): |
| setattr(hint, f"binding_{language}", True) |
| hint.all_service = True |
| |
| # c affected |
| if p.startswith("bindings/c/"): |
| hint.binding_c = True |
| hint.binding_go = True |
| hint.all_service = True |
| |
| # cpp affected |
| if p.startswith("bindings/cpp/"): |
| hint.binding_cpp = True |
| hint.all_service = True |
| |
| # go affected |
| if p.startswith(".github/scripts/test_go_binding"): |
| hint.binding_go = True |
| hint.all_service = True |
| |
| # bin affected |
| for bin in BIN: |
| if p.startswith(f"bin/{bin}"): |
| setattr(hint, f"bin_{bin}", True) |
| hint.all_service = True |
| |
| # integration affected |
| for integration in INTEGRATIONS: |
| if p.startswith(f"integrations/{integration}"): |
| setattr(hint, f"integration_{integration}", True) |
| hint.all_service = True |
| |
| # core service affected |
| match = re.search(r"core/src/services/([^/]+)/", p) |
| if match: |
| hint.core = True |
| for language in LANGUAGE_BINDING: |
| setattr(hint, f"binding_{language}", True) |
| for bin in BIN: |
| setattr(hint, f"bin_{bin}", True) |
| for integration in INTEGRATIONS: |
| setattr(hint, f"integration_{integration}", True) |
| hint.services.add(match.group(1)) |
| |
| # core test affected |
| match = re.search(r".github/services/([^/]+)/", p) |
| if match: |
| hint.core = True |
| for language in LANGUAGE_BINDING: |
| setattr(hint, f"binding_{language}", True) |
| for bin in BIN: |
| setattr(hint, f"bin_{bin}", True) |
| for integration in INTEGRATIONS: |
| setattr(hint, f"integration_{integration}", True) |
| hint.services.add(match.group(1)) |
| |
| # fixture affected |
| match = re.search(r"fixtures/([^/]+)/", p) |
| if match: |
| hint.core = True |
| for language in LANGUAGE_BINDING: |
| setattr(hint, f"binding_{language}", True) |
| for bin in BIN: |
| setattr(hint, f"bin_{bin}", True) |
| for integration in INTEGRATIONS: |
| setattr(hint, f"integration_{integration}", True) |
| hint.services.add(match.group(1)) |
| |
| return hint |
| |
| |
| # `unique_cases` is used to only one setup for each service. |
| # |
| # We need this because we have multiple setups for each service, and they have already been |
| # tested by `core` workflow. So we can only test unique setup for each service for bindings. |
| # |
| # We make sure that we return the first setup for each service in alphabet order. |
| def unique_cases(cases): |
| ucases = {} |
| for case in cases: |
| service = case["service"] |
| if service not in ucases: |
| ucases[service] = case |
| |
| # Convert the dictionary back to a list if needed |
| return list(ucases.values()) |
| |
| |
| def generate_core_cases( |
| cases: list[dict[str, str]], hint: Hint |
| ) -> list[dict[str, str]]: |
| # Always run all tests if it is a push event. |
| if os.getenv("GITHUB_IS_PUSH") == "true": |
| return cases |
| |
| # Return empty if core is False |
| if not hint.core: |
| return [] |
| |
| # Return all services if all_service is True |
| if hint.all_service: |
| return cases |
| |
| # Filter all cases that not shown un in changed files |
| cases = [v for v in cases if v["service"] in hint.services] |
| return cases |
| |
| |
| def generate_language_binding_cases( |
| cases: list[dict[str, str]], hint: Hint, language: str |
| ) -> list[dict[str, str]]: |
| cases = unique_cases(cases) |
| |
| # Disable aliyun_drive case for every language. |
| # |
| # This is because aliyun_drive has a speed limit and tests may not be stable enough. |
| # Bindings may be treated as parallel requests, so we need to disable it for all languages. |
| cases = [v for v in cases if v["service"] != "aliyun_drive"] |
| |
| # Remove hdfs cases for java and go. |
| if language == "java": |
| cases = [v for v in cases if v["service"] != "hdfs"] |
| |
| if os.getenv("GITHUB_IS_PUSH") == "true": |
| return cases |
| |
| # Return empty if this binding is False |
| if not getattr(hint, f"binding_{language}"): |
| return [] |
| |
| # Return all services if all_service is True |
| if hint.all_service: |
| return cases |
| |
| # Filter all cases that not shown up in changed files |
| cases = [v for v in cases if v["service"] in hint.services] |
| return cases |
| |
| |
| def generate_bin_cases( |
| cases: list[dict[str, str]], hint: Hint, bin: str |
| ) -> list[dict[str, str]]: |
| # Return empty if this bin is False |
| if not getattr(hint, f"bin_{bin}"): |
| return [] |
| |
| cases = unique_cases(cases) |
| |
| if bin == "ofs": |
| supported_services = ["fs", "s3"] |
| cases = [v for v in cases if v["service"] in supported_services] |
| |
| # Return all services if all_service is True |
| if hint.all_service: |
| return cases |
| |
| # Filter all cases that not shown up in changed files |
| cases = [v for v in cases if v["service"] in hint.services] |
| |
| return cases |
| |
| |
| def generate_integration_cases( |
| cases: list[dict[str, str]], hint: Hint, integration: str |
| ) -> list[dict[str, str]]: |
| # Return empty if this integration is False |
| if not getattr(hint, f"integration_{integration}"): |
| return [] |
| |
| cases = unique_cases(cases) |
| |
| if integration == "object_store": |
| supported_services = ["fs", "s3"] |
| cases = [v for v in cases if v["service"] in supported_services] |
| |
| # Return all services if all_service is True |
| if hint.all_service: |
| return cases |
| |
| # Filter all cases that not shown up in changed files |
| cases = [v for v in cases if v["service"] in hint.services] |
| |
| return cases |
| |
| |
| def plan(changed_files: list[str]) -> dict[str, Any]: |
| cases = provided_cases() |
| hint = calculate_hint(changed_files) |
| |
| core_cases = generate_core_cases(cases, hint) |
| |
| jobs = { |
| "components": { |
| "core": False, |
| }, |
| "core": [], |
| } |
| |
| if len(core_cases) > 0: |
| jobs["components"]["core"] = True |
| jobs["core"].append({"os": "ubuntu-latest", "cases": core_cases}) |
| |
| # fs is the only services need to run upon windows, let's hard code it here. |
| if "fs" in [v["service"] for v in core_cases]: |
| jobs["core"].append( |
| { |
| "os": "windows-latest", |
| "cases": [ |
| {"setup": "local_fs", "service": "fs", "feature": "services-fs"} |
| ], |
| } |
| ) |
| for language in LANGUAGE_BINDING: |
| jobs[f"binding_{language}"] = [] |
| jobs["components"][f"binding_{language}"] = False |
| language_cases = generate_language_binding_cases(cases, hint, language) |
| if len(language_cases) > 0: |
| jobs["components"][f"binding_{language}"] = True |
| jobs[f"binding_{language}"].append( |
| {"os": "ubuntu-latest", "cases": language_cases} |
| ) |
| if language == "go": |
| # Add fs service to ensure the go binding works on Windows and macOS. |
| jobs[f"binding_{language}"].append( |
| { |
| "os": "windows-latest", |
| "cases": [ |
| {"setup": "local_fs", "service": "fs", "feature": "services-fs"} |
| ], |
| } |
| ) |
| jobs[f"binding_{language}"].append( |
| { |
| "os": "macos-latest", |
| "cases": [ |
| {"setup": "local_fs", "service": "fs", "feature": "services-fs"} |
| ], |
| } |
| ) |
| |
| for bin in BIN: |
| jobs[f"bin_{bin}"] = [] |
| jobs["components"][f"bin_{bin}"] = False |
| bin_cases = generate_bin_cases(cases, hint, bin) |
| if len(bin_cases) > 0: |
| jobs["components"][f"bin_{bin}"] = True |
| jobs[f"bin_{bin}"].append({"os": "ubuntu-latest", "cases": bin_cases}) |
| |
| for integration in INTEGRATIONS: |
| jobs[f"integration_{integration}"] = [] |
| jobs["components"][f"integration_{integration}"] = False |
| integration_cases = generate_integration_cases(cases, hint, integration) |
| if len(integration_cases) > 0: |
| jobs["components"][f"integration_{integration}"] = True |
| jobs[f"integration_{integration}"].append( |
| {"os": "ubuntu-latest", "cases": integration_cases} |
| ) |
| |
| return jobs |
| |
| |
| if __name__ == "__main__": |
| changed_files = sys.argv[1:] |
| result = plan(changed_files) |
| print(json.dumps(result)) |