| #!/usr/bin/env python3 | |
| # /// script | |
| # requires-python = ">=3.11" | |
| # dependencies = [ | |
| # "GitPython", | |
| # "PyGithub", | |
| # "rich-click", | |
| # "packaging", | |
| # "rich", | |
| # ] | |
| # /// | |
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, | |
| # software distributed under the License is distributed on an | |
| # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY | |
| # KIND, either express or implied. See the License for the | |
| # specific language governing permissions and limitations | |
| # under the License. | |
| # This tool is based on the Spark merge_spark_pr script: | |
| # https://github.com/apache/spark/blob/master/dev/merge_spark_pr.py | |
| from __future__ import annotations | |
| import re | |
| import sys | |
| from collections import Counter, defaultdict | |
| from typing import TYPE_CHECKING | |
| import git | |
| import rich_click as click | |
| from github import Auth, Github | |
| from packaging import version | |
| from rich.console import Console | |
| from rich.progress import Progress | |
| if TYPE_CHECKING: | |
| from github.Issue import Issue | |
| GIT_COMMIT_FIELDS = ["id", "author_name", "author_email", "date", "subject", "body"] | |
| GIT_LOG_FORMAT = "%x1f".join(["%h", "%an", "%ae", "%ad", "%s", "%b"]) + "%x1e" | |
| pr_title_re = re.compile(r".*\((#[0-9]{1,6})\)$") | |
| STATUS_COLOR_MAP = { | |
| "Closed": "yellow", | |
| "Merged": "green", | |
| "Open": "red", | |
| } | |
| DEFAULT_SECTION_NAME = "Uncategorized" | |
| def get_commits_between( | |
| repo, | |
| previous_version, | |
| target_version, | |
| files=None, | |
| ): | |
| log_args = [f"--format={GIT_LOG_FORMAT}", previous_version + ".." + target_version] | |
| if files: | |
| log_args.append("--") | |
| log_args.append(" ".join(files)) | |
| log = repo.git.log(*log_args) | |
| log = log.strip("\n\x1e").split("\x1e") | |
| log = [row.strip().split("\x1f") for row in log] | |
| return [dict(zip(GIT_COMMIT_FIELDS, row)) for row in log] | |
| def style_issue_status(status): | |
| if status in STATUS_COLOR_MAP: | |
| return click.style(status[:6].ljust(6), STATUS_COLOR_MAP[status]) | |
| return status[:6].ljust(6) | |
| def get_issue_type(issue): | |
| label_prefix = "type:" | |
| issue_type = DEFAULT_SECTION_NAME | |
| if issue.labels: | |
| for label in issue.labels: | |
| if label.name.startswith(label_prefix): | |
| return label.name.replace(label_prefix, "").strip() | |
| if label.name == "changelog:skip": | |
| return "(skip)" | |
| return issue_type | |
| def build_main_commits_cache(repo: git.Repo, issue_numbers: set[int]) -> dict[int, str]: | |
| """Build a cache of PR number -> main branch commit SHA with a single git log operation""" | |
| cache = {} | |
| if not issue_numbers: | |
| return cache | |
| try: | |
| # Single git log to get all main branch commits | |
| # Use a reasonable range to avoid scanning entire history | |
| try: | |
| # Try to get commits since last major version (should cover most PR ranges) | |
| log_output = repo.git.log("origin/main", "--format=%H %s", "--since=1 year ago") | |
| except Exception: | |
| # Fallback to last 2000 commits if date fails | |
| log_output = repo.git.log("origin/main", "--format=%H %s", "-2000") | |
| # Use regex to find all PR numbers in commit messages at once | |
| import re | |
| pr_pattern = r"\(#(\d+)\)$" # PR number at end of commit message | |
| for commit_line in log_output.splitlines(): | |
| if not commit_line: | |
| continue | |
| # Find PR number at end of commit message | |
| match = re.search(pr_pattern, commit_line) | |
| if match: | |
| pr_number = int(match.group(1)) | |
| if pr_number in issue_numbers: | |
| commit_sha = commit_line.split(" ")[0] | |
| cache[pr_number] = commit_sha | |
| except Exception: | |
| # Fallback to empty cache if git operation fails | |
| pass | |
| return cache | |
| def get_commit_in_main_associated_with_pr(repo: git.Repo, issue: Issue, main_commits_cache: dict[int, str]) -> str | None: | |
| """For a PR, find the associated merged commit & return its SHA""" | |
| if issue.pull_request: | |
| # Use cache (should be pre-populated) | |
| return main_commits_cache.get(issue.number) | |
| return None | |
| def build_cherrypicked_cache(repo: git.Repo, issue_numbers: list[int], previous_version: str | None = None) -> dict[int, bool]: | |
| """Build a cache of which issues are cherry-picked by doing a single git log operation""" | |
| cache = {num: False for num in issue_numbers} | |
| if not issue_numbers: | |
| return cache | |
| # Get all commits in range and process them | |
| log_args = ["--format=%H %s"] | |
| if previous_version: | |
| log_args.append(previous_version + "..") | |
| try: | |
| log_output = repo.git.log(*log_args) | |
| # Use regex to find all PR numbers in the entire log output at once | |
| import re | |
| pr_pattern = r"\(#(\d+)\)" | |
| for commit_line in log_output.splitlines(): | |
| # Find all PR numbers in this commit | |
| matches = re.findall(pr_pattern, commit_line) | |
| for match in matches: | |
| issue_num = int(match) | |
| if issue_num in cache: | |
| cache[issue_num] = True | |
| except Exception: | |
| # Fallback to individual checks if batch fails | |
| pass | |
| return cache | |
| def is_cherrypicked(repo: git.Repo, issue: Issue, previous_version: str | None = None) -> bool: | |
| """Check if a given issue is cherry-picked in the current branch or not""" | |
| log_args = ["--format=%H %s", f"--grep=(#{issue.number})"] | |
| if previous_version: | |
| log_args.append(previous_version + "..") | |
| log_output = repo.git.log(*log_args) | |
| for commit_line in log_output.splitlines(): | |
| # Check if the PR number exists in the commit message (handles cherry-picked commits with multiple PR numbers) | |
| if commit_line and f"(#{issue.number})" in commit_line: | |
| return True | |
| return False | |
| def is_pr(issue: Issue) -> bool: | |
| return "apache/airflow/pull/" in issue.html_url | |
| def files_touched(repo, commit_id: str) -> list[str]: | |
| real_commit = repo.commit(commit_id) | |
| return real_commit.stats.files.keys() | |
| def is_core_commit(files: list[str]) -> bool: | |
| # We list out things that _aren't_ core files, | |
| # and we want to know if the commit changes anything | |
| # outside of these designated-non-core files | |
| non_core_files = [ | |
| # Providers | |
| "airflow/providers/", | |
| "airflow/provider.yaml.schema.json", | |
| "docs/apache-airflow-providers-", | |
| # chart | |
| "chart/", | |
| "docs/helm-chart/", | |
| "clients", | |
| # non-released docs | |
| "COMMITTERS.rst", | |
| "contributing_docs/", | |
| "INTHEWILD.md", | |
| "INSTALL", | |
| "README.md", | |
| "images/", | |
| "codecov.yml", | |
| "kubernetes-tests/", | |
| ".github/", | |
| ".pre-commit-config.yaml", | |
| "yamllint-config.yml", | |
| ".markdownlint.yml", | |
| "tests/", | |
| "dev/", | |
| "Dockerfile", | |
| "Dockerfile.ci", | |
| ".hadolint.yaml", | |
| "scripts/", | |
| "docs/build_docs.py", | |
| "docs/start_doc_server.sh", | |
| ".devcontainer/", | |
| "docker-context-files/", | |
| # Misc | |
| ".dockerignore", | |
| "docs/spelling_wordlist.txt", | |
| "docs/integration-logos/*", | |
| "docs/exts/", | |
| "docs/docker-stack", | |
| "docs/README.rst", | |
| "docker-tests/", | |
| "kubernetes-tests/", | |
| "helm-tests/", | |
| ".asf.yaml", | |
| ".mailmap", | |
| "breeze-legacy", | |
| "breeze-complete", | |
| ".rat-excludes", | |
| ".gitattributes", | |
| ".gitpod.yml", | |
| "generated/", | |
| ] | |
| for file in files: | |
| for ignore in non_core_files: | |
| if file.startswith(ignore): | |
| break | |
| # Handle renaming. Form: {old_name => new_name}somename.py | |
| if file.startswith("{"): | |
| new_files = file[1:].split(" => ") | |
| if any(n.strip().startswith(ignore) for n in new_files): | |
| break | |
| else: | |
| return True | |
| return False | |
| def print_changelog(sections): | |
| for section, lines in sections.items(): | |
| if section == "(skip)": | |
| continue | |
| print(section) | |
| print('"' * len(section)) | |
| for line in lines: | |
| print("-", line) | |
| print() | |
| @click.group() | |
| def cli(): | |
| r""" | |
| This tool should be used by Airflow Release Manager to verify what GitHub issues | |
| were merged in the current working branch. | |
| airflow-github compare <target_version> <github_token> | |
| """ | |
| @cli.command(short_help="Compare a GitHub target version against git merges") | |
| @click.argument("target_version") | |
| @click.argument("github-token", envvar="GITHUB_TOKEN") | |
| @click.option( | |
| "--previous-version", | |
| "previous_version", | |
| help="Specify the previous tag on the working branch to limit" | |
| " searching for few commits to find the cherry-picked commits", | |
| ) | |
| @click.option("--unmerged", "show_uncherrypicked_only", help="Show unmerged PRs only", is_flag=True) | |
| @click.option("--show-commits", help="Show commit SHAs (default: on, off when --unmerged)", is_flag=True, default=None) | |
| def compare(target_version, github_token, previous_version=None, show_uncherrypicked_only=False, show_commits=None): | |
| # Set smart defaults | |
| if show_commits is None: | |
| show_commits = not show_uncherrypicked_only # Default off for --unmerged | |
| repo = git.Repo(".", search_parent_directories=True) | |
| github_handler = Github(auth=Auth.Token(github_token)) | |
| # Fetch PRs and Issues separately, with merged PRs identified upfront | |
| merged_prs: list[Issue] = list( | |
| github_handler.search_issues( | |
| f'repo:apache/airflow milestone:"Airflow {target_version}" is:pull-request is:merged' | |
| ) | |
| ) | |
| closed_prs: list[Issue] = list( | |
| github_handler.search_issues( | |
| f'repo:apache/airflow milestone:"Airflow {target_version}" is:pull-request is:closed -is:merged' | |
| ) | |
| ) | |
| open_prs: list[Issue] = list( | |
| github_handler.search_issues( | |
| f'repo:apache/airflow milestone:"Airflow {target_version}" is:pull-request is:open' | |
| ) | |
| ) | |
| # Skip fetching issues if we only care about unmerged PRs | |
| if show_uncherrypicked_only: | |
| issues = [] | |
| else: | |
| issues: list[Issue] = list( | |
| github_handler.search_issues( | |
| f'repo:apache/airflow milestone:"Airflow {target_version}" is:issue' | |
| ) | |
| ) | |
| # Create a merge status lookup | |
| pr_merge_status_cache = {} | |
| for pr in merged_prs: | |
| pr_merge_status_cache[pr.number] = True | |
| for pr in closed_prs: | |
| pr_merge_status_cache[pr.number] = False | |
| # Open PRs are neither merged nor closed, so we don't need to cache them | |
| milestone_issues = merged_prs + closed_prs + open_prs + issues | |
| num_cherrypicked = 0 | |
| num_uncherrypicked = Counter() | |
| # :<18 says left align, pad to 18, :>6 says right align, pad to 6 | |
| # :<50.50 truncates after 50 chars | |
| # !s forces as string | |
| if show_commits: | |
| formatstr = ( | |
| "{number:>6} | {typ!s:<5} | {changelog!s:<13} | {status!s} " | |
| "| {title:<83.83} | {merged:<6} | {commit:>7.7} | {url}" | |
| ) | |
| header_fields = { | |
| "number": "NUMBER", | |
| "typ": "TYPE", | |
| "changelog": "CHANGELOG", | |
| "status": "STATUS".ljust(6), | |
| "title": "TITLE", | |
| "merged": "CHERRY", | |
| "commit": "COMMIT", | |
| "url": "URL", | |
| } | |
| else: | |
| formatstr = ( | |
| "{number:>6} | {typ!s:<5} | {changelog!s:<13} | {status!s} " | |
| "| {title:<95.95} | {merged:<6} | {url}" | |
| ) | |
| header_fields = { | |
| "number": "NUMBER", | |
| "typ": "TYPE", | |
| "changelog": "CHANGELOG", | |
| "status": "STATUS".ljust(6), | |
| "title": "TITLE", | |
| "merged": "CHERRY", | |
| "commit": "", # Not used | |
| "url": "URL", | |
| } | |
| print(formatstr.format(**header_fields)) | |
| milestone_issues = sorted( | |
| milestone_issues, key=lambda x: x.closed_at if x.closed_at else x.created_at, reverse=True | |
| ) | |
| # Build caches for performance optimization | |
| issue_numbers = [issue.number for issue in milestone_issues if is_pr(issue)] | |
| # Convert to set for O(1) lookups in cache building | |
| issue_numbers_set = set(issue_numbers) | |
| # Build all caches upfront with batch operations | |
| if show_commits: | |
| main_commits_cache = build_main_commits_cache(repo, issue_numbers_set) | |
| else: | |
| main_commits_cache = {} | |
| cherrypicked_cache = build_cherrypicked_cache(repo, issue_numbers, previous_version) | |
| for issue in milestone_issues: | |
| issue_is_pr = is_pr(issue) | |
| # Determine status - differentiate between Closed and Merged for PRs using cache | |
| if issue_is_pr and issue.state == "closed": | |
| is_merged = pr_merge_status_cache.get(issue.number, False) | |
| status = "Merged" if is_merged else "Closed" | |
| else: | |
| status = issue.state.capitalize() | |
| # Checks if commit was cherrypicked into branch using cache | |
| if issue_is_pr: | |
| is_cherry_picked = cherrypicked_cache.get(issue.number, False) | |
| else: | |
| is_cherry_picked = is_cherrypicked(repo, issue, previous_version) | |
| if is_cherry_picked: | |
| num_cherrypicked += 1 | |
| if show_uncherrypicked_only: | |
| continue | |
| cherrypicked = click.style("Yes".ljust(6), "green") | |
| elif not issue_is_pr and show_uncherrypicked_only: | |
| # Don't show issues when looking for unmerged PRs | |
| continue | |
| elif issue_is_pr: | |
| num_uncherrypicked[status] += 1 | |
| cherrypicked = click.style("No".ljust(6), "red") | |
| else: | |
| cherrypicked = "" | |
| fields = dict( | |
| number=issue.number, | |
| typ="PR" if issue_is_pr else "Issue", | |
| changelog=get_issue_type(issue) if issue_is_pr else "", | |
| status=style_issue_status(status), | |
| title=issue.title, | |
| url=issue.html_url, | |
| ) | |
| # Only get commit info if we're showing commits | |
| if show_commits: | |
| commit_in_main = get_commit_in_main_associated_with_pr(repo, issue, main_commits_cache) | |
| fields["commit"] = commit_in_main if commit_in_main else "" | |
| print(formatstr.format(**fields, merged=cherrypicked)) | |
| print( | |
| f"Commits on branch: {num_cherrypicked:d}, {sum(num_uncherrypicked.values()):d} " | |
| f"({dict(num_uncherrypicked)}) yet to be cherry-picked" | |
| ) | |
| @cli.command(short_help="Build a CHANGELOG grouped by GitHub Issue type") | |
| @click.argument("previous_version") | |
| @click.argument("target_version") | |
| @click.argument("github-token", envvar="GITHUB_TOKEN") | |
| @click.option("--disable-progress-bar", is_flag=True, help="Disable the progress bar") | |
| def changelog(previous_version, target_version, github_token, disable_progress_bar): | |
| repo = git.Repo(".", search_parent_directories=True) | |
| # Get a list of issues/PRs that have been committed on the current branch. | |
| log = get_commits_between(repo, previous_version, target_version) | |
| if disable_progress_bar: | |
| print(f"Processing {len(log)} commits") | |
| gh = Github(auth=Auth.Token(github_token)) | |
| gh_repo = gh.get_repo("apache/airflow") | |
| sections = defaultdict(list) | |
| with open("RELEASE_NOTES.rst") as file: | |
| existing_pr_nos = re.findall(r"\(#(\d+)\)", file.read()) | |
| console = Console(width=180) | |
| with Progress(console=console, disable=disable_progress_bar) as progress: | |
| task = progress.add_task("Processing commits from changelog", total=len(log)) | |
| for commit in progress.track(log, description="Processing commits from changelog"): | |
| tickets = pr_title_re.findall(commit["subject"]) | |
| if tickets: | |
| issue = gh_repo.get_issue(number=int(tickets[0][1:])) | |
| issue_type = get_issue_type(issue) | |
| files = files_touched(repo, commit["id"]) | |
| if str(issue.number) in existing_pr_nos: | |
| print(f"Skipping {issue.number} as it's already in RELEASE_NOTES.rst") | |
| continue | |
| if is_core_commit(files): | |
| sections[issue_type].append(commit["subject"]) | |
| progress.update(task, advance=1) | |
| else: | |
| sections[DEFAULT_SECTION_NAME].append(commit["subject"]) | |
| progress.update(task, advance=1) | |
| console.print("\n") | |
| print_changelog(sections) | |
| @cli.command(short_help="Find merged PRs that still need to be categorized for the changelog") | |
| @click.argument("previous_version") | |
| @click.argument("target_version") | |
| @click.option("--show-skipped", is_flag=True) | |
| @click.option("--show-files", is_flag=True) | |
| @click.argument("github-token", envvar="GITHUB_TOKEN") | |
| def needs_categorization(previous_version, target_version, show_skipped, show_files, github_token): | |
| repo = git.Repo(".", search_parent_directories=True) | |
| log = get_commits_between(repo, previous_version, target_version) | |
| gh = Github(auth=Auth.Token(github_token)) | |
| gh_repo = gh.get_repo("apache/airflow") | |
| for commit in log: | |
| tickets = pr_title_re.findall(commit["subject"]) | |
| if tickets: | |
| issue = gh_repo.get_issue(number=int(tickets[0][1:])) | |
| issue_type = get_issue_type(issue) | |
| if issue_type == DEFAULT_SECTION_NAME: | |
| files = files_touched(repo, commit["id"]) | |
| if is_core_commit(files): | |
| print(f"{commit['subject']}: {issue.html_url}") | |
| if show_files: | |
| for f in files: | |
| print(f"\t{f}") | |
| elif show_skipped: | |
| print(f"**** {commit['subject']}: Skipping - not a core commit") | |
| if show_files: | |
| for f in files: | |
| print(f"\t{f}") | |
| else: | |
| print(f"Commit '{commit['id']}' is missing PR number: {commit['subject']}") | |
| @cli.command( | |
| name="api-clients-policy", | |
| help="Compare two airflow core release tags and determine if API clients need to be released.", | |
| ) | |
| @click.argument("previous_version") | |
| @click.argument("target_version") | |
| def api_clients_policy(previous_version, target_version): | |
| p_version = version.parse(previous_version) | |
| t_version = version.parse(target_version) | |
| if p_version.major != t_version.major: | |
| print("This is a major release, API clients should also be released.") | |
| return | |
| if p_version.minor != t_version.minor: | |
| print("This is a minor release, API clients should also be released.") | |
| return | |
| if p_version == t_version: | |
| print("Both versions are identical") | |
| return | |
| repo = git.Repo(".", search_parent_directories=True) | |
| log = get_commits_between( | |
| repo, | |
| previous_version, | |
| target_version, | |
| files=[f"{repo.working_dir}/airflow-core/src/airflow/api_fastapi/core_api/openapi/v2-rest-api-generated.yaml"], | |
| ) | |
| clients_need_release = False | |
| for commit in log: | |
| if "update airflow version to" not in commit["subject"].lower(): | |
| clients_need_release = True | |
| print(f"Commit '{commit['id']}' updated the OpenAPI spec, PR number: {commit['subject']}") | |
| if clients_need_release: | |
| print(f"API clients need to be released because the API spec has changed since '{previous_version}'") | |
| else: | |
| print( | |
| "API clients don't need to be released because the API spec hasn't changed between those two " | |
| "patch versions" | |
| ) | |
| if __name__ == "__main__": | |
| import doctest | |
| (failure_count, test_count) = doctest.testmod() | |
| if failure_count: | |
| sys.exit(-1) | |
| try: | |
| cli() | |
| except Exception: | |
| raise |