scripts/ci/prek/check_eval_hash.py

#!/usr/bin/env python
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
"""Verify the skill-eval ran against the current guidance files.

``dev/skill-evals/eval.py`` records a hash of its inputs in
``dev/skill-evals/last-eval-hash.txt`` after every completed run. This hook
recomputes the hash from the current file contents and fails when AGENTS.md
or the eval cases changed without a re-run. The hash proves the eval *ran*
on this exact content — not that all cases passed.

The comparison is pure file content — no git state — so the hook behaves
identically at commit time, at push time, and in CI (``prek --all-files``).
"""

from __future__ import annotations

import hashlib
import sys
from pathlib import Path

REPO_ROOT = Path(__file__).resolve().parents[3]
AGENTS_FILE = REPO_ROOT / "AGENTS.md"
CASES_DIR = REPO_ROOT / "dev" / "skill-evals" / "cases"
HASH_FILE = REPO_ROOT / "dev" / "skill-evals" / "last-eval-hash.txt"

HASH_FILE_HEADER = """\
# Generated by dev/skill-evals/eval.py — do not edit or resolve conflicts by hand.
# Run `prek run run-skill-eval --hook-stage manual --all-files` to regenerate.
"""


def compute_guidance_hash(agents_file: Path, cases_dir: Path) -> str:
    """Hash the skill-eval inputs: AGENTS.md plus every case file.

    Labels are hashed alongside contents so renaming a case file changes
    the hash. SKILL.md files are intentionally excluded — no skill cases
    exist yet; extend the inputs when per-skill cases land (runs with
    ``SKILL_NAME=...``).
    """
    digest = hashlib.sha256()
    entries = [("AGENTS.md", agents_file)]
    entries += [(f"cases/{path.name}", path) for path in sorted(cases_dir.glob("*.yaml"))]
    for label, path in entries:
        digest.update(label.encode())
        digest.update(b"\0")
        digest.update(path.read_bytes())
        digest.update(b"\0")
    return digest.hexdigest()


def read_recorded_hash(hash_file: Path) -> str | None:
    """Return the hash recorded in ``hash_file``, or None if absent."""
    if not hash_file.is_file():
        return None
    for line in hash_file.read_text().splitlines():
        stripped = line.strip()
        if stripped and not stripped.startswith("#"):
            return stripped
    return None


def write_recorded_hash(value: str, hash_file: Path) -> None:
    """Record ``value`` with a regeneration note, like breeze's output-commands-hash.txt."""
    hash_file.write_text(f"{HASH_FILE_HEADER}{value}\n")


def main() -> int:
    current = compute_guidance_hash(AGENTS_FILE, CASES_DIR)
    recorded = read_recorded_hash(HASH_FILE)
    if current == recorded:
        return 0
    print(
        "AGENTS.md or dev/skill-evals/cases/ changed without re-running the"
        " skill-eval\n(dev/skill-evals/last-eval-hash.txt does not match).\n"
        "\n"
        "  Run:  prek run run-skill-eval --hook-stage manual --all-files\n"
        "        then commit the updated dev/skill-evals/last-eval-hash.txt\n"
        "\n"
        "  WIP commit: SKIP=check-eval-hash git commit ...\n"
        "  Can't run the eval? Ask a maintainer to run it and push the updated\n"
        "  hash file to your PR branch (see dev/skill-evals/README.md).",
        file=sys.stderr,
    )
    return 1


if __name__ == "__main__":
    raise SystemExit(main())