scripts/tests/ci/prek/test_check_eval_hash.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
from __future__ import annotations

from unittest import mock

import pytest
from ci.prek.check_eval_hash import (
    compute_guidance_hash,
    main,
    read_recorded_hash,
    write_recorded_hash,
)


@pytest.fixture
def guidance(tmp_path):
    """A tmp AGENTS.md and cases dir mirroring the eval input layout."""
    agents_file = tmp_path / "AGENTS.md"
    agents_file.write_text("golden rule\n")
    cases_dir = tmp_path / "cases"
    cases_dir.mkdir()
    (cases_dir / "newsfragment.yaml").write_text("- description: case\n")
    return agents_file, cases_dir


def test_compute_guidance_hash_is_deterministic(guidance):
    agents_file, cases_dir = guidance

    assert compute_guidance_hash(agents_file, cases_dir) == compute_guidance_hash(agents_file, cases_dir)


@pytest.mark.parametrize(
    "mutate",
    [
        pytest.param(
            lambda agents_file, cases_dir: agents_file.write_text("golden rule, revised\n"),
            id="agents-content",
        ),
        pytest.param(
            lambda agents_file, cases_dir: (cases_dir / "newsfragment.yaml").write_text(
                "- description: revised case\n"
            ),
            id="case-content",
        ),
        pytest.param(
            lambda agents_file, cases_dir: (cases_dir / "routing.yaml").write_text(
                "- description: another case\n"
            ),
            id="case-added",
        ),
        pytest.param(
            lambda agents_file, cases_dir: (cases_dir / "newsfragment.yaml").rename(
                cases_dir / "renamed.yaml"
            ),
            id="case-renamed",
        ),
    ],
)
def test_compute_guidance_hash_changes_when_inputs_change(guidance, mutate):
    agents_file, cases_dir = guidance
    before = compute_guidance_hash(agents_file, cases_dir)

    mutate(agents_file, cases_dir)

    assert compute_guidance_hash(agents_file, cases_dir) != before


def test_read_recorded_hash_returns_none_when_missing(tmp_path):
    assert read_recorded_hash(tmp_path / "absent.txt") is None


def test_write_then_read_roundtrip(tmp_path):
    hash_file = tmp_path / "last-eval-hash.txt"

    write_recorded_hash("abc123", hash_file)

    assert read_recorded_hash(hash_file) == "abc123"
    assert hash_file.read_text().startswith("# Generated by dev/skill-evals/eval.py")
    assert hash_file.read_text().endswith("abc123\n")


def test_main_passes_when_hash_matches(guidance, tmp_path, capsys):
    agents_file, cases_dir = guidance
    hash_file = tmp_path / "last-eval-hash.txt"
    write_recorded_hash(compute_guidance_hash(agents_file, cases_dir), hash_file)

    with (
        mock.patch("ci.prek.check_eval_hash.AGENTS_FILE", agents_file),
        mock.patch("ci.prek.check_eval_hash.CASES_DIR", cases_dir),
        mock.patch("ci.prek.check_eval_hash.HASH_FILE", hash_file),
    ):
        assert main() == 0

    assert capsys.readouterr().err == ""


@pytest.mark.parametrize("record_stale_hash", [True, False], ids=["stale-hash", "missing-file"])
def test_main_fails_on_mismatch_or_missing_record(guidance, tmp_path, capsys, record_stale_hash):
    agents_file, cases_dir = guidance
    hash_file = tmp_path / "last-eval-hash.txt"
    if record_stale_hash:
        write_recorded_hash(compute_guidance_hash(agents_file, cases_dir), hash_file)
        agents_file.write_text("changed after the recorded run\n")

    with (
        mock.patch("ci.prek.check_eval_hash.AGENTS_FILE", agents_file),
        mock.patch("ci.prek.check_eval_hash.CASES_DIR", cases_dir),
        mock.patch("ci.prek.check_eval_hash.HASH_FILE", hash_file),
    ):
        assert main() == 1

    err = capsys.readouterr().err
    assert "prek run run-skill-eval" in err
    assert "SKIP=check-eval-hash" in err