blob: 373e6c8b16c37f325b1c09c77dcb8f239ad0851d [file]
#!/usr/bin/env python3
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""
Compares the latest test run's timings against the repository's recorded history and reports modules that got slower or faster.
Full docs: push-timings.md
"""
from __future__ import annotations
import argparse
import json
import os
import statistics
import sys
from collections import defaultdict
from pathlib import Path
# The shared loader (_juneau.py) sits beside this script; appended, so no script here shadows an installed module.
if str(Path(__file__).resolve().parent) not in sys.path:
sys.path.append(str(Path(__file__).resolve().parent))
import _juneau # noqa: E402
REPO_ROOT = Path(__file__).resolve().parent.parent
THRESHOLD_ENV = "JUNEAU_PUSH_TIMING_THRESHOLD"
MIN_SECONDS_ENV = "JUNEAU_PUSH_TIMING_MIN_SECONDS"
DEFAULT_THRESHOLD = 0.20
DEFAULT_MIN_SECONDS = 5.0
MARKS = {"slower": "⚠", "faster": "↓", None: " "}
MODULE_ROWS = "module"
BUCKET_SERIES = "surefire_s." # surefire_s.<bucket>: {module: summed Surefire seconds}
def load_records(path: Path) -> list[dict]:
"""The JSON objects of a JSONL file, oldest first; a missing file or a malformed line is skipped."""
if not path.exists():
return []
rows = []
for line in path.read_text(encoding="utf-8").splitlines():
if not line.strip():
continue
try:
row = json.loads(line)
except json.JSONDecodeError:
continue
if isinstance(row, dict):
rows.append(row)
return rows
def rows_from_records(records: list[dict]) -> list[dict]:
"""
Today's rows from test.py's `tests` records: {run_id, passed, module, execution, wallclock_s} per module and bucket,
plus reactor/suite from wall_s. The run_id is the record's ts and commit; passed is its ok. Only records with the
newest record's sig count (_juneau.same_shape_runs).
"""
tests = _juneau.same_shape_runs(records)
rows = []
for record in tests:
base = {"run_id": f"{record.get('ts', '')}-{record.get('commit', '')}", "passed": record.get("ok") is True}
for name, values in sorted((record.get("series") or {}).items()):
if name.startswith(BUCKET_SERIES) and isinstance(values, dict):
rows += [{**base, "module": module, "execution": name[len(BUCKET_SERIES):], "wallclock_s": seconds}
for module, seconds in sorted(values.items())]
if record.get("wall_s") is not None:
rows.append({**base, "module": "reactor", "execution": "suite", "wallclock_s": record["wall_s"]})
return rows
def _store(repo):
jr = _juneau.load()[0]
return jr, jr.HistoryStore(Path(repo), "test")
def history_off() -> bool:
"""True when JUNEAU_TERMINAL_HISTORY=0: the store neither records nor reads."""
return not _juneau.load()[0].history_enabled()
def load_history(repo=None, path=None) -> list[dict]:
"""The rows (rows_from_records) of the repo's test.py history, or of a records file when path is given."""
if path is not None:
return rows_from_records(load_records(Path(path)))
return rows_from_records(_store(repo or REPO_ROOT)[1].read())
def report_path(repo=None, path=None) -> Path:
"""Where the full table goes: test-timings.txt beside the store's test.jsonl, or <stem>-timings.txt beside a
records file."""
if path is not None:
path = Path(path)
return path.with_name(f"{path.stem}-timings.txt")
return _store(repo or REPO_ROOT)[1].path.with_name("test-timings.txt")
def fmt_seconds(value: float) -> str:
return f"{value:.1f}s"
def fmt_delta(value: float) -> str:
sign = "+" if value >= 0 else ""
return f"{sign}{value * 100:.1f}%"
def _env_float(name: str, default: float) -> float:
"""$name as a float, or default when it is unset or not a number (the CLI must always exit 0)."""
try:
return float(os.environ.get(name) or default)
except ValueError:
return default
def threshold() -> float:
"""The relative margin, as a fraction: $JUNEAU_PUSH_TIMING_THRESHOLD, default 0.20."""
return _env_float(THRESHOLD_ENV, DEFAULT_THRESHOLD)
def min_seconds() -> float:
"""The absolute margin in seconds: $JUNEAU_PUSH_TIMING_MIN_SECONDS, default 5."""
return _env_float(MIN_SECONDS_ENV, DEFAULT_MIN_SECONDS)
def latest_run(records: list[dict]) -> tuple[list[dict], str | None]:
"""The latest run's passing rows, or an empty list and the reason there are none."""
if not records:
return [], "no timing history yet"
run_id = records[-1].get("run_id")
if not run_id:
return [], "missing run metadata in latest record"
rows = [r for r in records if r.get("run_id") == run_id and r.get("passed", True)
and r.get("execution") != MODULE_ROWS]
if not rows:
return [], "latest run did not produce successful timing rows"
return rows, None
def classify(seconds: float, baseline: float | None, rel: float, abs_seconds: float) -> str | None:
""""slower" or "faster" when the entry moved by more than both margins, else None."""
if not baseline:
return None
diff = seconds - baseline
if abs(diff) / baseline > rel and abs(diff) > abs_seconds:
return "slower" if diff > 0 else "faster"
return None
def analyse(records: list[dict], window: int = 20) -> list[dict]:
"""
One entry per (module, execution) in the latest run, sorted by module and execution. Each is a dict with module,
execution, seconds, baseline (the median of the last `window` earlier passing runs, None on a first run), delta (a
fraction, 0 without a baseline), flag ("slower", "faster" or None) and line (its row in the table).
"""
rows, _ = latest_run(records)
if not rows:
return []
run_id = rows[0]["run_id"]
prior: dict[tuple[str, str], list[float]] = defaultdict(list)
for row in records:
if row.get("run_id") != run_id and row.get("passed", True):
prior[(row.get("module", "?"), row.get("execution", "?"))].append(float(row.get("wallclock_s", 0.0)))
rel, abs_seconds = threshold(), min_seconds()
entries = []
for row in sorted(rows, key=lambda r: (r.get("module", ""), r.get("execution", ""))):
module, execution = row.get("module", "?"), row.get("execution", "?")
seconds = float(row.get("wallclock_s", 0.0))
earlier = prior.get((module, execution), [])
baseline = statistics.median(earlier[-window:]) if earlier else None
delta = (seconds - baseline) / baseline if baseline else 0.0
if baseline is None:
line = f"{module}/{execution}: {fmt_seconds(seconds)} (first run, no baseline)"
else:
line = (f"{module}/{execution}: {fmt_seconds(seconds)} (median {fmt_seconds(baseline)}, "
f"delta {fmt_delta(delta)}, {seconds - baseline:+.1f}s)")
entries.append({"module": module, "execution": execution, "seconds": seconds, "baseline": baseline,
"delta": delta, "flag": classify(seconds, baseline, rel, abs_seconds), "line": line})
return entries
def counts(entries: list[dict]) -> str:
"""
"<n> slower than 20% and 5s · <k> faster · <m> without a baseline"; the faster and baseline parts are left out
when zero.
"""
slower = sum(1 for e in entries if e["flag"] == "slower")
faster = sum(1 for e in entries if e["flag"] == "faster")
first_runs = sum(1 for e in entries if e["baseline"] is None)
parts = [f"{slower} slower than {threshold():.0%} and {min_seconds():g}s"]
if faster:
parts.append(f"{faster} faster")
if first_runs:
parts.append(f"{first_runs} without a baseline")
return " · ".join(parts)
def write_report(entries: list[dict], path: Path, window: int = 20) -> None:
"""Write the full table to path, replacing it: every entry with its mark, then the counts."""
lines = [f"Push-timing report (last {window} runs in this repo)"]
lines += [f" {MARKS[e['flag']]} {e['line']}" for e in entries] or [" no successful timing rows in the latest run"]
lines.append(counts(entries))
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description="Print push timing regression summary.")
source = parser.add_mutually_exclusive_group()
source.add_argument("--repo", help="The repo whose run history to read (default: this script's repo)")
source.add_argument("--history", help="A test.py history records file to read instead of the repo's store")
parser.add_argument("--window", type=int, default=20, help="Rolling median window per execution")
parser.add_argument("--report", help="Where to write the full table (default: test-timings.txt beside the store)")
args = parser.parse_args(argv)
history = Path(args.history).expanduser() if args.history else None
repo = Path(args.repo).expanduser() if args.repo else REPO_ROOT
if history is None and history_off():
print("📊 Push-timing report: run history is off (JUNEAU_TERMINAL_HISTORY=0).")
return 0
records = load_history(repo, history)
_, reason = latest_run(records)
if reason:
print(f"📊 Push-timing report: {reason}.")
return 0
runs = len({row["run_id"] for row in records if row.get("passed", True)})
if runs < 2:
print(f"📊 Push-timing report: {_juneau.no_baseline_line(runs)}.")
return 0
entries = analyse(records, args.window)
report = Path(args.report).expanduser() if args.report else report_path(repo, history)
write_report(entries, report, args.window)
print(f"📊 Push-timing report (last {args.window} runs in this repo): {counts(entries)} → {report}")
for entry in entries:
if entry["flag"] == "slower":
print(f" ⚠ {entry['line']}")
return 0
if __name__ == "__main__":
raise SystemExit(main())