blob: bbd9347e4f2eb89c3dfbbad6dc1ac4014cef67bc [file]
#!/usr/bin/env python3
# ***************************************************************************************************************************
# * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file
# * distributed with this work for additional information regarding copyright ownership. The ASF licenses this file
# * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
# * with the License. You may obtain a copy of the License at
# *
# * http://www.apache.org/licenses/LICENSE-2.0
# *
# * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
# * specific language governing permissions and limitations under the License.
# ***************************************************************************************************************************
"""
Builds Juneau, runs the Maven tests, then checks timings, runs the scripts' own pytest suite, and optionally the browser tests.
Full docs: test.md
"""
USAGE = """
Build and test helper script for Apache Juneau.
Usage:
./scripts/test.py [options]
Options:
--build-only, -b Only build (skip tests)
--test-only, -t Only run tests (no build)
--full, -f Clean build + run tests (default)
--verbose, -v Accepted for compatibility; Maven output always streams live
--no-container Exclude @Tag("container") tests
--timing-log <path> Ignored, with a warning (timings are in the run history now); removed in a later release
--enforce-perf Hard-fail on a perf-guard breach (see "Perf guard" below)
--rebaseline Rewrite perf-baseline.txt's suite line and [modules] from this repo's test.py run
history (~/.cache/juneau-terminal/history), then exit (no build)
--window <n> --rebaseline: runs per key to take the median of (default 20); a module whose window
spreads by more than 50% ((max - min) / min) takes the p75 instead (the suite keeps the median)
--min-runs <n> --rebaseline: skip keys with fewer runs than this (default 5)
--js-tests Also run the headless-browser JS harness (mvn -Pjs-tests, *_BrowserTest in
juneau-console-views, juneau-petstore-core and juneau-terminal-region). Fails if Node/npm are missing.
--no-js-tests Never run the JS harness (overrides auto-detect)
--warn-traces Report stack traces in the test output as a warning instead of failing (see "Trace guard" below)
--console <mode> Maven output on the console: full (default), board or none
--detail <level> Console detail: summary, actionable (default), modules or all (JUNEAU_TERMINAL_DETAIL)
--full-log <path> Also write every byte Maven prints to this file (appended)
--board-log <path> Also write the board one-line-per-event stream to this file (appended)
--profile <module> Run one-shot JFR profile for module tests
--help, -h Show this help message
Environment:
JUNEAU_MVN_WRAPPER Optional prefix for every mvn command (e.g. a lock script that serializes
concurrent runs on one checkout).
Python Tests:
After the Maven tests pass, scripts/tests (the tooling's own pytest suite) runs as its own row. The launcher is
`python3 -m pytest` when pytest is importable, else `uv run --native-tls --with pytest --with pyyaml` when uv is on the PATH; with neither,
the step prints a notice and skips. Nothing is installed.
Browser Tests:
The *_BrowserTest classes (juneau-console-views, juneau-petstore-core, juneau-terminal-region) are excluded from the main test
pass, so it reports no skips for them; this step runs them with -Pjs-tests, as CI's js-tests and
js-tests-petstore and js-tests-terminal jobs do. When neither
--js-tests nor --no-js-tests is given, this script enables them automatically if any .js, .cjs, .css or .ftl
file under a src/ tree differs from origin/master (committed, uncommitted or untracked). If Node/npm
are not on the PATH, an auto-enabled run prints a notice and skips; an explicit --js-tests fails instead.
The first run downloads Playwright + Chromium (needs network); later runs reuse target/js.
Trace guard:
After the other steps pass, a Traces row scans what they printed (read back from the run's full log) for stack
traces, and fails on any whose header line is not matched by scripts/trace-allowlist.txt. The log exists with
--console board or none; with the default --console full there is none, so the row is skipped with a warning.
--warn-traces reports traces as a warning instead; an allowlist error still fails. The report of every trace,
allowed or not, goes to target/trace-guard.txt.
Perf guard:
After the tests pass, the guard compares the measured wall clock of `mvn test` against 'suite' in the
project-root perf-baseline.txt, and each module's reactor wall clock (from Maven's Reactor Summary, mapped to
module paths by each pom's own <name>) against its [modules] entry.
The suite breaches above baseline Ɨ (1 + JUNEAU_CI_PERF_THRESHOLD, default 0.20). A module breaches only when
it is over that AND more than JUNEAU_CI_PERF_MIN_SECONDS (default 3) over its baseline. A module with no entry
warns and never fails. The report goes to target/perf-guard.txt.
Without --enforce-perf the check runs in warn-only mode (prints results, never exits non-zero for a perf
breach). Each run's history record keeps the suite time and every module's reactor time (the reactor_s
series), which --rebaseline turns into new baselines.
"""
import argparse
import math
import os
import re
import shlex
import shutil
import statistics
import subprocess
import sys
import time
import xml.etree.ElementTree as ET
from datetime import datetime
from pathlib import Path
# The shared loader (_juneau.py) sits beside this script; appended, so no script here shadows an installed module.
if str(Path(__file__).resolve().parent) not in sys.path:
sys.path.append(str(Path(__file__).resolve().parent))
import _juneau # noqa: E402
REPO_ROOT = Path(__file__).resolve().parent.parent
TRACE_ALLOWLIST = Path(__file__).resolve().parent / "trace-allowlist.txt"
TRACE_REPORT = REPO_ROOT / "target" / "trace-guard.txt"
_step_n = 0
def next_step_number():
"""Top-level step numbers continue after JUNEAU_TERMINAL_N_BASE, which push.py sets for the steps it spawns us for."""
global _step_n
_step_n += 1
return int(os.environ.get("JUNEAU_TERMINAL_N_BASE", "0")) + _step_n
def say(text, level="info"):
"""Script status text: through juneau_terminal (a filtered note once a console view owns the screen)."""
_juneau.load()[0].say(text, level)
_SHELL_OPERATORS = {"&&", "||", ";", "|"}
def _join_words(words):
"""shlex.join, except that a word that is exactly a shell chaining operator stays an operator."""
return " ".join(w if w in _SHELL_OPERATORS else shlex.quote(w) for w in words)
def mvn_wrapper_prefix():
"""JUNEAU_MVN_WRAPPER as a shell-safe command prefix, or "" when unset.
The value is a command plus arguments, not a shell fragment: it is split like a shell would split it
(shlex.split, so a pre-quoted `'/a b/lock.sh' --x` works) and re-joined with each word quoted (shlex.join),
so a path with spaces or `$`/`;` is never interpreted; a word that is exactly `&&`, `||`, `;` or `|` stays an operator. An unquoted path with spaces is also accepted: the
longest leading run of words that names an existing file is taken as the executable. A value that does not
parse (unbalanced quote) is passed through unchanged."""
wrapper = os.environ.get("JUNEAU_MVN_WRAPPER", "").strip()
if not wrapper:
return ""
# Candidate executables end at a whitespace boundary (or the end); the longest one that is a file wins.
ends = [m.start() for m in re.finditer(r"\s+", wrapper)] + [len(wrapper)]
for end in reversed(ends):
head = wrapper[:end]
if os.path.isfile(head):
try:
rest = shlex.split(wrapper[end:])
except ValueError:
rest = wrapper[end:].split()
return _join_words([head, *rest])
try:
return _join_words(shlex.split(wrapper))
except ValueError:
return wrapper
def wrap_mvn_command(cmd):
"""cmd prefixed with the quoted JUNEAU_MVN_WRAPPER, when one is set."""
prefix = mvn_wrapper_prefix()
return f"{prefix} {cmd}" if prefix else cmd
def run_command(cmd, step_id=None, title=None, label=None, summarize=None):
"""Run a command, streaming its output live, and return exit code and full output.
A command with a step_id runs as that step through juneau_terminal's run_tool whatever the output routing, so its row
is recorded in the run history; with markers off and the default sinks its output is the same as a plain run's.
Without a step_id (or without juneau_terminal) it runs under Popen, as it always has. summarize(text), when given,
runs after the command exits while its row is still open (see _record_buckets); never on the Popen path.
"""
project_root = REPO_ROOT
# Optional command prefix (e.g. a lock script that serializes concurrent mvn runs on one checkout).
if cmd.startswith("mvn "):
cmd = wrap_mvn_command(cmd)
say(f"Running: {cmd}")
say("-" * 80) # NOSONAR python:S1192 - the same banner rule as the separator in _main; a constant hides it
run = _juneau.load()[0]
if step_id:
# Through a shell, like the default path, so the quoted JUNEAU_MVN_WRAPPER prefix keeps its meaning.
result = run.run_tool(["/bin/sh", "-c", cmd], "maven", step_id, title or step_id, n=next_step_number(),
cwd=str(project_root), capture=True, label=label, summarize=summarize)
return result.exit, result.output
# Echo each line as it arrives so a long or hung build is visible, and keep it for parse_test_results().
lines = []
with subprocess.Popen(cmd, shell=True, cwd=str(project_root), stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
text=True, errors="replace", bufsize=1) as proc:
for line in proc.stdout:
sys.stdout.write(line)
sys.stdout.flush()
lines.append(line)
return proc.returncode, "".join(lines)
def git_value(args):
try:
result = subprocess.run(["git", *args], cwd=str(Path(__file__).parent.parent), capture_output=True, text=True, check=True)
return result.stdout.strip()
except Exception:
return "unknown"
def parse_test_results(output):
matches = list(re.finditer(r"\[ERROR\]\s+Tests run:\s+(\d+),\s+Failures:\s+(\d+),\s+Errors:\s+(\d+)", output))
if matches:
match = matches[-1]
total = int(match.group(1))
failures = int(match.group(2))
errors = int(match.group(3))
return total, failures, errors
return None, None, None
# ─── Per-module timing/perf subsystem ────────────────────────────────
#
# Tests live in each module's own src/test/java and
# report into that module's target/surefire-reports/. The helpers below discover
# ALL such report dirs under the reactor (not just juneau-integration-tests's), attribute each
# Surefire XML to its OWNING module (the parent of target/surefire-reports), and
# bucket each test class.
#
# Module key = the module's path relative to the repo root (e.g. "juneau-integration-tests",
# "juneau-core/juneau-junit5"; MAY contain '/').
# Bucket ∈ {core, container.springboot, container.jetty, container.tomcat}.
#
# juneau-integration-tests additionally splits its reports into surefire-reports/{core,container}/
# via two Surefire executions; the recursive XML scan below transparently handles
# both that nested layout and the standard flat layout every migrated module uses.
# Container annotation marker -> bucket. Checked against the test class source.
CONTAINER_MARKERS = (
("SpringbootTest", "container.springboot"),
("JettyMicroserviceTest", "container.jetty"),
("TomcatMicroserviceTest", "container.tomcat"),
)
def classify_bucket(module_dir: Path, class_name: str) -> str:
"""Bucket a test class as core or a container.* flavor via source-annotation inspection."""
source = module_dir / "src" / "test" / "java" / Path("/".join(class_name.split("."))).with_suffix(".java")
if source.exists():
try:
content = source.read_text(encoding="utf-8")
for marker, bucket in CONTAINER_MARKERS:
if marker in content:
return bucket
return "core"
except OSError:
pass
# Source unreadable (rare): fall back to a name-based heuristic, defaulting to core.
lowered = class_name.lower()
if "springboot" in lowered:
return "container.springboot"
if "tomcat" in lowered:
return "container.tomcat"
if "jetty" in lowered:
return "container.jetty"
return "core"
def aggregate_module_reports(module_dir: Path, reports_dir: Path) -> dict:
"""Return {bucket: {'tests': int, 'seconds': float}} for one module's surefire-reports tree."""
buckets: dict = {}
for xml_file in sorted(reports_dir.rglob("TEST-*.xml")):
try:
root = ET.parse(xml_file).getroot()
except (ET.ParseError, OSError):
continue
bucket = classify_bucket(module_dir, root.attrib.get("name", ""))
stats = buckets.setdefault(bucket, {"tests": 0, "seconds": 0.0})
stats["tests"] += int(root.attrib.get("tests", 0))
stats["seconds"] += float(root.attrib.get("time", 0.0))
return buckets
def discover_module_stats(repo_root: Path) -> dict:
"""Map module-key -> {bucket -> {tests, seconds}} for every surefire-reports dir under the reactor."""
modules: dict = {}
for reports_dir in sorted(repo_root.rglob("target/surefire-reports")):
if not reports_dir.is_dir():
continue
module_dir = reports_dir.parent.parent
module_key = module_dir.relative_to(repo_root).as_posix()
buckets = aggregate_module_reports(module_dir, reports_dir)
if buckets:
modules[module_key] = buckets
return modules
# A Reactor Summary row, read the way juneau_terminal's Maven parser reads it (MavenParser.SUMMARY_ROW in juneau_terminal.py).
SECTION = re.compile(r"^\[([\w.-]+)\]$")
REACTOR_SUMMARY = re.compile(r"^\[INFO\] Reactor Summary")
SUMMARY_ROW = re.compile(r"^\[INFO\] (.+?)(?: \.+)? (SUCCESS|FAILURE|SKIPPED)(?: \[\s*([^\]]+?)\s*\])?\s*$")
ANSI = re.compile(r"\x1b\[[0-9;]*m")
POM_SKIP_DIRS = {"target", "node_modules", "src", "build-overlay"}
def _summary_seconds(text):
"""'7.513 s' -> 7.513, '02:45 min' -> 165.0, '01:28 h' -> 5280.0 (as juneau_terminal's _seconds)."""
text = text.strip()
m = re.match(r"^(\d+):(\d+)\s*(min|h)?$", text)
if m:
a, b, unit = int(m.group(1)), int(m.group(2)), m.group(3)
return float(a * 3600 + b * 60 if unit == "h" else a * 60 + b)
m = re.match(r"^([\d.]+)\s*s?$", text)
return float(m.group(1)) if m else 0.0
def reactor_times(output: str) -> dict:
"""{module name: seconds} from the Reactor Summary of one Maven run's output. Only SUCCESS rows count."""
times: dict = {}
in_summary = False
for raw in output.splitlines():
line = ANSI.sub("", raw)
if REACTOR_SUMMARY.match(line):
in_summary = True
continue
row = SUMMARY_ROW.match(line) if in_summary else None
if row and row.group(2) == "SUCCESS" and row.group(3):
times[row.group(1).strip()] = _summary_seconds(row.group(3))
return times
def module_names(repo_root: Path) -> tuple:
"""({project name: module path}, [ambiguous names]) for every pom.xml outside target/, node_modules/, src/,
build-overlay/ (helper poms packaged into a zip, not reactor modules) and dot-directories.
The name is the project's own <name> (its <artifactId> when it has none), never its parent's; it is what the
Reactor Summary prints. A name two poms share is left out and listed as ambiguous. The root pom is skipped.
"""
paths: dict = {}
ambiguous: set = set()
for dirpath, dirnames, filenames in os.walk(repo_root):
dirnames[:] = sorted(d for d in dirnames if d not in POM_SKIP_DIRS and not d.startswith("."))
directory = Path(dirpath)
if "pom.xml" not in filenames or directory == repo_root:
continue
try:
root = ET.parse(directory / "pom.xml").getroot()
except (ET.ParseError, OSError):
continue
ns = root.tag[:root.tag.index("}") + 1] if root.tag.startswith("{") else ""
node = root.find(f"{ns}name")
if node is None or not (node.text or "").strip():
node = root.find(f"{ns}artifactId")
if node is None or not (node.text or "").strip():
continue
name = node.text.strip()
if name in paths or name in ambiguous:
paths.pop(name, None)
ambiguous.add(name)
else:
paths[name] = directory.relative_to(repo_root).as_posix()
return paths, sorted(ambiguous)
def module_times(output: str, repo_root: Path) -> tuple:
"""({module path: reactor seconds}, [names with no single pom]) for one `mvn test` run's output."""
reactor = reactor_times(output)
if not reactor:
return {}, []
names, _ = module_names(repo_root)
times = {names[name]: seconds for name, seconds in reactor.items() if name in names}
return times, sorted(name for name in reactor if name not in names)
def read_baselines(baseline_file: Path) -> dict:
"""Parse perf-baseline.txt: 'suite' from anywhere, and every '<module path> = <seconds>' under [modules].
Entries are '<key> = <seconds>', optionally followed by a '#' comment. A '[name]' line starts a section. Blank
lines, comments and every other section (e.g. [observability], which scripts/microbench.py reads) are ignored.
"""
result: dict = {}
if not baseline_file.exists():
return result
section = None
for line in baseline_file.read_text(encoding="utf-8").splitlines():
stripped = line.split("#", 1)[0].strip()
heading = SECTION.match(stripped)
if heading:
section = heading.group(1)
continue
if "=" not in stripped:
continue
key, _, value = stripped.partition("=")
key = key.strip()
if key != "suite" and section != "modules":
continue
try:
result[key] = float(value.strip().split()[0])
except (ValueError, IndexError):
continue
return result
PERF_REPORT = Path(__file__).resolve().parent.parent / "target" / "perf-guard.txt"
PERF_BASELINE = Path(__file__).resolve().parent.parent / "perf-baseline.txt"
class _PerfOutput:
"""Where PERF-GUARD lines go: notes on the Perf row when a console view owns the screen, else print. Every line
also goes to the report file."""
def __init__(self, report_file):
run = _juneau.load()[0]
self.run = run if run.console_owns_screen() else None
self.report_file = Path(report_file)
self.lines = []
def line(self, text, level="info"):
self.lines.append(text)
if self.run is not None:
self.run.note(level, text, step="perf")
else:
print(text)
def report_only(self, text):
self.lines.append(text)
def write(self):
self.report_file.parent.mkdir(parents=True, exist_ok=True)
self.report_file.write_text("\n".join(self.lines) + "\n", encoding="utf-8")
def run_perf_guard(test_elapsed: float, baseline_file: Path, module_times: dict, enforce: bool, report_file=None,
unmapped=()) -> int:
"""Perf guard: the suite's wall clock, and each module's reactor wall clock (module_times, from module_times()).
A breach is a warn note, or an error with enforce=True. Modules with no baseline collapse into one info note.
The full report goes to report_file (target/perf-guard.txt); the Perf row shows the counts and its path.
Returns exit code: 0 = pass or warn-only mode, 1 = threshold breached with enforce=True.
"""
run = _juneau.load()[0]
with run.row("perf", "Perf", label="Perf") as row:
out = _PerfOutput(report_file or PERF_REPORT)
code, summary = _perf_guard(test_elapsed, baseline_file, module_times, unmapped, enforce, out)
row.summary = summary
if code != 0:
row.status = "fail"
return code
def _perf_guard(test_elapsed, baseline_file, module_times, unmapped, enforce, out):
"""The check itself. Returns (exit code, Perf row summary)."""
tolerance = float(os.environ.get("JUNEAU_CI_PERF_THRESHOLD", "0.20"))
min_seconds = float(os.environ.get("JUNEAU_CI_PERF_MIN_SECONDS", "3"))
breach = "error" if enforce else "warn"
baselines = read_baselines(baseline_file)
if not baselines:
out.line(f"PERF-GUARD: baseline file not found or empty ({baseline_file}); skipping check.", "warn")
out.write()
return 0, f"no baseline file → {out.report_file}"
pct = f"±{tolerance * 100:.0f}%"
over = 0
# 1) Suite-level guard.
suite_baseline = baselines.get("suite")
if suite_baseline is not None:
threshold = suite_baseline * (1 + tolerance)
if test_elapsed > threshold:
out.line(
f"PERF-GUARD FAIL [suite]: tests took {test_elapsed:.1f}s "
f"(baseline {suite_baseline}s, tolerance {pct}, threshold {threshold:.1f}s).\n"
f" If intentional, bump the 'suite' entry in perf-baseline.txt (project root).", breach)
over += 1
else:
out.line(f"PERF-GUARD OK [suite]: tests took {test_elapsed:.1f}s (baseline {suite_baseline}s, tolerance {pct}).")
else:
out.line("PERF-GUARD WARN [suite]: no 'suite' baseline configured; skipping wall-clock check.")
# 2) Modules: over by the tolerance AND by min_seconds, so a short module's jitter never trips it.
margins = f"{pct} and {min_seconds:g}s"
checked = 0
regressions = 0
new_modules = []
for key in sorted(module_times):
actual = module_times[key]
baseline = baselines.get(key)
if baseline is None:
new_modules.append(key)
continue
checked += 1
threshold = max(baseline * (1 + tolerance), baseline + min_seconds)
if actual > threshold:
out.line(
f"PERF-GUARD FAIL [{key}]: {actual:.1f}s "
f"(baseline {baseline}s, tolerance {margins}, threshold {threshold:.1f}s).\n"
f" If intentional, run `python3 scripts/test.py --rebaseline` or bump '{key}' in perf-baseline.txt "
f"(project root).", breach)
regressions += 1
over += 1
lines = [f"PERF-GUARD WARN [{key}]: {module_times[key]:.1f}s — no baseline entry yet (new/unknown module; not failing)."
for key in new_modules]
if out.run is None:
for text in lines:
out.line(text)
else:
for text in lines:
out.report_only(text)
if new_modules:
out.line(f"PERF-GUARD: {len(new_modules)} module(s) have no baseline entry yet: "
+ ", ".join(new_modules))
if unmapped:
out.report_only(f"PERF-GUARD: {len(unmapped)} reactor module(s) not mapped to a pom (name shared or missing): "
+ ", ".join(unmapped))
out.line(
f"PERF-GUARD SUMMARY: {checked} module(s) checked, {regressions} regression(s), "
f"{len(new_modules)} new/unknown."
)
if over and not enforce:
out.line("PERF-GUARD: warn-only mode (pass --enforce-perf to hard-fail on breach).")
out.write()
if over:
summary = f"⚠ {over} over baseline" + (f" · {len(new_modules)} without one" if new_modules else "")
else:
summary = f"āœ“ {checked} checked" + (f" Ā· {len(new_modules)} without a baseline" if new_modules else "")
summary += f" → {out.report_file}"
return (1 if over and enforce else 0), summary
SUITE_LINE = re.compile(r"^suite\s*=")
OUTLIER_FACTOR = 3
BIMODAL_SPREAD = 0.5
def baseline_series(records) -> dict:
"""{key: [seconds...]} in run order over the passing `tests` records with the newest record's sig (its command
shape): 'suite' from each record's wall_s, and a module path from each entry of its reactor_s series."""
series: dict = {}
for record in _juneau.same_shape_runs(records):
if record.get("ok") is not True:
continue
if isinstance(record.get("wall_s"), (int, float)):
series.setdefault("suite", []).append(float(record["wall_s"]))
reactor = (record.get("series") or {}).get("reactor_s")
for key, seconds in (reactor.items() if isinstance(reactor, dict) else ()):
if isinstance(seconds, (int, float)) and not isinstance(seconds, bool):
series.setdefault(key, []).append(float(seconds))
return series
def rebaseline_value(values: list, window: int, bimodal_p75: bool = False) -> float:
"""The median of the last `window` values, after dropping any over 3Ɨ their median, rounded up to 0.1s (a
baseline is a ceiling). With `bimodal_p75`, a series whose kept values spread by more than BIMODAL_SPREAD
((max - min) / min) takes its 75th percentile instead of the median."""
recent = values[-window:]
median = statistics.median(recent)
kept = [v for v in recent if v <= OUTLIER_FACTOR * median]
pick = statistics.median(kept)
# A module that flips between a fast and a slow mode (CPU contention from neighbouring modules) has a median in
# the fast mode, so the guard would fire on every slow-mode run; the p75 sits in the slow mode.
if bimodal_p75 and len(kept) >= 2 and (max(kept) - min(kept)) / min(kept) > BIMODAL_SPREAD:
pick = statistics.quantiles(kept, n=4, method="inclusive")[2]
return math.ceil(round(pick * 10, 6)) / 10
def rebaseline(baseline_file: Path, records, window: int = 20, min_runs: int = 5) -> int:
"""Rewrite perf-baseline.txt's suite line and [modules] section from the history medians; print what changed.
records are test.py's run-history records (HistoryStore(repo, "test").read()); baseline_series keeps the passing
`tests` ones of the newest record's sig. When that command shape has fewer than min_runs passing runs, prints a
"no baseline for this command shape" line and writes nothing.
A key with fewer than min_runs passing runs in the window is skipped and keeps its value; a [modules] entry with
no history at all is removed. Everything outside the suite line and the [modules] body is kept byte-for-byte.
Runs no build. Returns 0, or 1 when there is no history or no [modules] line (nothing is written then).
"""
text = baseline_file.read_text(encoding="utf-8") if baseline_file.exists() else ""
lines = text.splitlines(keepends=True)
start = next((i for i, line in enumerate(lines) if line.strip() == "[modules]"), None)
if start is None:
print(f"āŒ {baseline_file} has no [modules] section line; nothing rewritten.")
return 1
series = baseline_series(records)
if not series:
print("āŒ no passing runs in test.py's run history; nothing rewritten.")
return 1
runs = sum(1 for r in _juneau.same_shape_runs(records) if r.get("ok") is True)
if runs < min_runs:
print(f"šŸ“ {_juneau.no_baseline_line(runs)}; nothing rewritten.")
return 1
end = next((i for i in range(start + 1, len(lines)) if SECTION.match(lines[i].strip())), len(lines))
# Comments and blank lines just above the next section (its banner) belong to it, not to [modules].
while end > start + 1 and (not lines[end - 1].strip() or lines[end - 1].lstrip().startswith("#")):
end -= 1
before = read_baselines(baseline_file)
fresh, skipped = {}, {}
for key, values in series.items():
if len(values[-window:]) < min_runs:
skipped[key] = len(values[-window:])
else:
fresh[key] = rebaseline_value(values, window, bimodal_p75=key != "suite")
modules = {k: v for k, v in before.items() if k != "suite" and k in skipped}
modules.update({k: v for k, v in fresh.items() if k != "suite"})
if not lines[start].endswith("\n"):
lines[start] += "\n"
body = [f"{key} = {modules[key]:g}\n" for key in sorted(modules)]
if end < len(lines) and lines[end].strip():
body.append("\n")
out = lines[:start + 1] + body + lines[end:]
if "suite" in fresh:
suite_line = f"suite = {fresh['suite']:g}\n"
at = next((i for i, line in enumerate(out) if SUITE_LINE.match(line)), None)
if at is None:
out.insert(start, suite_line)
else:
out[at] = suite_line
baseline_file.write_text("".join(out), encoding="utf-8")
after = read_baselines(baseline_file)
keys = sorted(set(before) | set(after), key=lambda k: (k != "suite", k))
changed = [k for k in keys if k in before and k in after and before[k] != after[k]]
added = [k for k in keys if k not in before]
removed = [k for k in keys if k not in after]
print(f"šŸ“ Rebaselined {baseline_file} from the run history (last {window} passing runs): {len(changed)} changed, "
f"{len(added)} added, {len(removed)} removed, {len(skipped)} skipped (fewer than {min_runs} runs).")
for k in changed:
print(f" ~ {k}: {before[k]:g} → {after[k]:g}")
for k in added:
print(f" + {k}: {after[k]:g}")
for k in removed:
print(f" - {k}: {before[k]:g}")
for k in sorted(skipped):
print(f" ? {k}: {skipped[k]} run(s)")
return 0
# Reactor-level parallel *module* builds. -T1C runs one build thread per CPU core, so independent
# reactor modules (and their test forks) build concurrently, overlapping the otherwise-serial
# per-module test phases. Requires the juneau-distrib dependency:copy ordering fix (MDEP-187) to be
# -T-safe. This is parallel *module* execution only, NOT in-JVM concurrent test classes/methods.
PARALLELISM = "-T1C"
def build():
return run_command(f"mvn clean install {PARALLELISM} -DskipTests", "build", "Build", label="Compile")
_last_times = None # (module times, unmapped names) of the last test() run, from _record_buckets
def _record_buckets(text):
"""run_tool's summarize for the tests step, called while its row is open: one surefire_s.<bucket> series per bucket
with data ({module path: summed Surefire class seconds}), and reactor_s ({module path: reactor seconds}, the perf
guard's metric, also kept in _last_times for _main). Returns None, so the row keeps the parser's summary."""
global _last_times
run = _juneau.load()[0]
per_bucket = {}
for module, buckets in discover_module_stats(REPO_ROOT).items():
for bucket, stats in buckets.items():
per_bucket.setdefault(bucket, {})[module] = round(stats["seconds"], 3)
for bucket, values in sorted(per_bucket.items()):
run.history_series(f"surefire_s.{bucket}", values)
_last_times = module_times(text or "", REPO_ROOT)
if _last_times[0]:
run.history_series("reactor_s", _last_times[0])
return None
def test(no_container=False):
global _last_times
_last_times = None
cmd = f"mvn test {PARALLELISM} -Drat.skip=true"
if no_container:
cmd += " -DexcludedGroups=container"
return run_command(cmd, "tests", "Java Tests", label="Java Tests", summarize=_record_buckets)
# Set in the pytest child so a test that reaches script_tests() again cannot launch the suite recursively.
SCRIPT_TESTS_ACTIVE = "JUNEAU_SCRIPT_TESTS_ACTIVE"
def script_tests_launcher():
"""The command prefix that runs pytest, or None. `python3 -m pytest` when pytest is importable, else
`uv run --native-tls --with pytest --with pyyaml` when uv is on the PATH. Nothing is ever installed."""
python = shutil.which("python3") or sys.executable
try:
importable = subprocess.run([python, "-c", "import pytest"], capture_output=True).returncode == 0
except OSError:
importable = False
if importable:
return [python, "-m", "pytest"]
if shutil.which("uv"):
return ["uv", "run", "--native-tls", "--with", "pytest", "--with", "pyyaml", "pytest"]
return None
def script_tests_command(launcher):
return [*launcher, "-q", "-p", "no:cacheprovider", "scripts/tests"]
def script_tests(launcher=None, runner=None):
"""The scripts/tests pytest suite as its own row. Returns 0 on pass or skip, non-zero on failure."""
if os.environ.get(SCRIPT_TESTS_ACTIVE):
say("\nāš ļø Skipping Python Tests: already running inside them (no recursion).", "warn")
skip_py_tests_row("already running inside them")
return 0
launcher = launcher or script_tests_launcher()
if launcher is None:
say("\nāš ļø Skipping Python Tests: neither pytest nor uv found (python3 -m pytest / uv run --native-tls --with pytest --with pyyaml).", "warn")
skip_py_tests_row("neither pytest nor uv found")
return 0
cmd = script_tests_command(launcher)
say("\nšŸ Running Python Tests (scripts/tests)...")
say(f"Running: {' '.join(cmd)}")
say("-" * 80) # NOSONAR python:S1192 - the same banner rule as run_command
env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1", **{SCRIPT_TESTS_ACTIVE: "1"})
if runner is not None:
code = runner(cmd, env)
else:
run = _juneau.load()[0]
code = run.run_tool(cmd, "pytest", "py-tests", "Python Tests", n=next_step_number(), cwd=str(REPO_ROOT),
env=env, capture=True, label="Python Tests").exit
say("\nāœ… Python Tests passed!" if code == 0 else "\nāŒ Python Tests failed!", "info" if code == 0 else "error")
return code
# The modules whose *_BrowserTest classes the browser step runs, with the package-qualified -Dtest pattern that
# selects each module's own suite (the same patterns as CI's js-tests, js-tests-petstore and js-tests-terminal jobs; a bare
# '*_BrowserTest' would also match a sibling module's suite in the reactor). Those classes are excluded from the
# default surefire pass by each module's pom and enabled by its js-tests profile.
JS_TEST_MODULES = (
("juneau-rest/juneau-console-views", "org.apache.juneau.console.views.*_BrowserTest"),
("juneau-petstore/juneau-petstore-core", "org.apache.juneau.petstore.**.*_BrowserTest"),
("juneau-rest/juneau-terminal-region", "org.apache.juneau.terminal.region.*_BrowserTest"),
)
JS_SOURCE_SUFFIXES = (".js", ".cjs", ".css", ".ftl")
def is_js_source(path):
"""True for a .js/.css/.ftl file that lives under a src/ tree (any module)."""
p = path.replace("\\", "/")
parts = p.split("/")
return p.lower().endswith(JS_SOURCE_SUFFIXES) and "src" in parts[:-1]
def changed_files_vs_origin(repo_root=None, base="origin/master"):
"""Paths changed in HEAD, the working tree or untracked, relative to `base`. Empty list if git/base unavailable."""
repo_root = repo_root or Path(__file__).parent.parent
names = []
try:
for args in (["diff", "--name-only", base], ["ls-files", "--others", "--exclude-standard"]):
r = subprocess.run(["git", *args], cwd=str(repo_root), capture_output=True, text=True, check=True)
names.extend(line for line in r.stdout.splitlines() if line.strip())
except Exception:
return []
return names
def should_run_js_tests(js_flag, no_js_flag, changed_files):
"""Resolve (enabled, explicit). Explicit flags win (--no-js-tests over --js-tests); otherwise auto-detect."""
if no_js_flag:
return False, False
if js_flag:
return True, True
return any(is_js_source(f) for f in changed_files), False
def js_prereq_problem():
"""Return a description of the missing browser prerequisite, or None if Node and npm are on the PATH."""
missing = [tool for tool in ("node", "npm") if shutil.which(tool) is None]
if missing:
return f"{' and '.join(missing)} not found on the PATH (the -Pjs-tests profile needs Node and npm)"
return None
def js_tests_command():
"""One reactor run over every JS test module. .mvn/maven.config's --also-make builds the upstream modules from
the working tree, so the same command serves a run after this script's install and a --test-only run."""
modules = ",".join(module for module, _ in JS_TEST_MODULES)
patterns = ",".join(pattern for _, pattern in JS_TEST_MODULES)
return (f"mvn -Pjs-tests -pl {modules} test -Drat.skip=true "
f"-Dtest='{patterns}' -Dsurefire.failIfNoSpecifiedTests=false")
# Forks per module while the modules overlap; a lone petstore run takes the pom's default of 3.
JS_CONCURRENT_FORKS = 2
def js_tests_concurrent_command():
"""The same suites as js_tests_command, as one mvn per module so they overlap (one reactor runs them in dependency
order). Each `mvn -f` resolves its upstream modules from ~/.m2 (hence -o), so this is valid only straight after the
install step. Output streams live with a [module] prefix; the shell waits for every module and fails if any did,
and a module that left no exit status counts as a failure. Needs bash (PIPESTATUS)."""
runs = []
for i, (module, pattern) in enumerate(JS_TEST_MODULES):
runs.append(f"(mvn -o -Pjs-tests -f {module}/pom.xml test -Drat.skip=true -Dtest='{pattern}' "
f"-Dsurefire.failIfNoSpecifiedTests=false -Djuneau.jsTests.forks={JS_CONCURRENT_FORKS} 2>&1 "
f"| sed -u 's|^|[{module}] |' | tee target/js-tests-{i}.log; echo ${{PIPESTATUS[0]}} > target/js-tests-{i}.exit) &")
codes = " ".join(f"$(cat target/js-tests-{i}.exit 2>/dev/null || echo 1)" for i in range(len(JS_TEST_MODULES)))
return ("mkdir -p target && rm -f target/js-tests-*.exit && " + " ".join(runs)
+ f" wait ; for c in {codes}; do [ \"$c\" = 0 ] || exit 1; done")
def js_tests(installed=False):
"""The JS harness: the views, petstore and terminal browser suites. Straight after the install step the three
modules run at once (js_tests_concurrent_command); otherwise one sequential reactor run (js_tests_command), since
the upstream modules are then built from the working tree rather than read from ~/.m2."""
if not installed:
return run_command(js_tests_command(), "js-tests", "Browser Tests", label="Browser Tests")
cmd = f"/bin/bash -c {shlex.quote(js_tests_concurrent_command())}"
# Wraps the whole compound: run_command only prefixes commands that start with "mvn ".
cmd = wrap_mvn_command(cmd)
return run_command(cmd, "js-tests", "Browser Tests", label="Browser Tests")
#: test.py's top-level steps in run order, as (id, label). The labels are push.py's PLAN_LABELS for the same ids:
#: under push.py this script's pending rows stand in for the parent's.
PLAN = (("build", "Compile"), ("tests", "Java Tests"), ("perf", "Perf"), ("py-tests", "Python Tests"),
("js-tests", "Browser Tests"), ("traces", "Traces"))
def planned_steps(build_only, test_only, full, no_js_tests):
"""The steps this run will take, in order, as (id, label) pairs, for the console's pending rows and Overall
figure. --no-js-tests leaves Browser Tests out; a run-time skip (no JS changes) stays in the plan, and its skip
row is left out of the count."""
ids = {"build"} if build_only or full else set()
if test_only or full:
ids |= {"tests", "perf", "py-tests", "traces"} | (set() if no_js_tests else {"js-tests"})
return [(step_id, label) for step_id, label in PLAN if step_id in ids]
_js_skipped = False # the Browser Tests row was skipped this run, so JS-only allowlist entries cannot have matched
def run_trace_guard(since, warn_only, full_run):
"""The Traces row: the output of the steps since `since` (traces.mark(), taken before the build) must hold no stack
trace that is not in scripts/trace-allowlist.txt. warn_only (--warn-traces) downgrades a failure to a warning.
full_run lets the guard report allowlist entries that matched nothing. Returns 0 or 1."""
from juneau_terminal_scripts import traces
return traces.gate("traces", "Traces", TRACE_ALLOWLIST, since=since, warn_only=warn_only, full_run=full_run,
report=TRACE_REPORT, label="Traces")
def skip_py_tests_row(reason):
"""Draw the Python Tests row as skipped, with the reason as its text."""
with _juneau.load()[0].row("py-tests", "Python Tests", label="Python Tests") as row:
row.skip(reason)
def skip_js_tests_row(reason):
"""Draw the Browser Tests row as skipped, with the reason as its text."""
global _js_skipped
_js_skipped = True
with _juneau.load()[0].row("js-tests", "Browser Tests", label="Browser Tests") as row:
row.skip(reason)
def maybe_run_js_tests(js_flag, no_js_flag, changed_files, runner=None, installed=False):
"""Run the JS harness if enabled. Returns 0 on pass/skip, non-zero on failure. A skipped harness still gets its
row, marked skipped with the reason."""
enabled, explicit = should_run_js_tests(js_flag, no_js_flag, changed_files)
if not enabled:
skip_js_tests_row("--no-js-tests" if no_js_flag
else "no JS changes since origin/master (--js-tests to force)")
return 0
problem = js_prereq_problem()
if problem:
if explicit:
say(f"\nāŒ --js-tests requested but cannot run: {problem}.", "error")
return 1
say(f"\nāš ļø JS files changed, but skipping Browser Tests: {problem}. (CI will still run them.)", "warn")
skip_js_tests_row(problem.split(" (")[0])
return 0
say("\n🌐 Running Browser Tests (-Pjs-tests)..." + ("" if explicit else " (auto: JS/CSS/FTL files changed)"))
code, _ = (runner or (lambda: js_tests(installed)))()
say("\nāœ… Browser Tests passed!" if code == 0 else "\nāŒ Browser Tests failed!", "info" if code == 0 else "error")
return code
def module_surefire_extra_args(module_dir):
"""The module pom's own <surefire.extraArgs> value (e.g. the mockito agent), or "" when it sets none."""
match = re.search(r"<surefire\.extraArgs>(.*?)</surefire\.extraArgs>", (module_dir / "pom.xml").read_text(), re.S)
return match.group(1).strip() if match else ""
def profile_command(module, output_path):
"""The mvn command that profiles one module, run from that module's directory.
The jacoco agent is skipped, because its prepare-agent goal rewrites argLine after any -DargLine, and the JFR flags
ride on -Dsurefire.extraArgs (which the surefire argLine already appends) after the module's own value.
"""
module_dir = REPO_ROOT / module
jfr = f"-XX:FlightRecorderOptions:stackdepth=1024 -XX:StartFlightRecording=filename={output_path},settings=profile,dumponexit=true"
extra = " ".join(part for part in (module_surefire_extra_args(module_dir), jfr) if part)
mvn = f"mvn test -Drat.skip=true -Djacoco.skip=true {shlex.quote('-Dsurefire.extraArgs=' + extra)}"
return f"cd {shlex.quote(str(module_dir))} && {wrap_mvn_command(mvn)}"
def profile(module):
ts = datetime.now().strftime("%Y%m%d-%H%M%S")
profile_dir = (REPO_ROOT / "target" / "profile-results").resolve()
profile_dir.mkdir(parents=True, exist_ok=True)
safe_module = module.replace("/", "-")
# An absolute directory, so it does not resolve against each forked JVM's working directory. JFR writes one
# hotspot-pid-<pid>-id-<n>-<timestamp>.jfr into it per JVM, so executions and forks never overwrite each other.
output_dir = profile_dir / f"{safe_module}-{ts}"
output_dir.mkdir()
code, out = run_command(profile_command(module, output_dir), "profile", "Profile", label="Profile")
if code != 0:
return code, out
written = sorted(output_dir.glob("*.jfr"))
if not written:
say(f"\nāŒ mvn passed but no JFR file was written under {output_dir}.", "error")
return 1, out
for jfr_file in written:
say(f"\nāœ… JFR profile captured at {jfr_file}", "warn")
return code, out
def _main(): # NOSONAR python:S3776 -- Cognitive complexity is acceptable for this main function
parser = argparse.ArgumentParser(add_help=False)
parser.add_argument("--build-only", "-b", action="store_true")
parser.add_argument("--test-only", "-t", action="store_true")
parser.add_argument("--full", "-f", action="store_true")
parser.add_argument("--verbose", "-v", action="store_true")
parser.add_argument("--no-container", action="store_true")
parser.add_argument("--timing-log")
parser.add_argument("--enforce-perf", action="store_true")
parser.add_argument("--rebaseline", action="store_true")
parser.add_argument("--window", type=int, default=20)
parser.add_argument("--min-runs", type=int, default=5, dest="min_runs")
parser.add_argument("--js-tests", action="store_true", dest="js_tests")
parser.add_argument("--no-js-tests", action="store_true", dest="no_js_tests")
parser.add_argument("--warn-traces", action="store_true", dest="warn_traces")
parser.add_argument("--profile")
parser.add_argument("--console", choices=("full", "board", "none"))
parser.add_argument("--detail")
parser.add_argument("--full-log")
parser.add_argument("--board-log")
parser.add_argument("--help", "-h", action="store_true")
args, unknown = parser.parse_known_args()
if args.help:
print(USAGE)
return 0
if unknown:
print(f"Unknown option(s): {' '.join(unknown)}")
print(USAGE)
return 1
if args.timing_log:
say("⚠ --timing-log is ignored: timings are in the run history now (scripts/push-timings.py reads it). "
"The option goes away in a later release.", "warn")
if args.rebaseline:
return _rebaseline_from_history(args.window, args.min_runs)
# juneau_terminal reads its output routing from the environment; an argument beats a value already set there.
for flag, variable in ((args.console, "JUNEAU_TERMINAL_CONSOLE"), (args.full_log, "JUNEAU_TERMINAL_FULL_LOG"),
(args.board_log, "JUNEAU_TERMINAL_BOARD_LOG")):
if flag:
os.environ[variable] = flag
try:
_juneau.load()[0].export_detail(args.detail)
except ValueError as e:
print(e)
return 2
build_only = args.build_only
test_only = args.test_only
full = args.full or not (build_only or test_only or args.profile)
if args.profile:
exit_code, _ = profile(args.profile)
return exit_code
if build_only and test_only:
print("Cannot combine --build-only and --test-only")
return 1
if args.js_tests and args.no_js_tests:
print("Cannot combine --js-tests and --no-js-tests")
return 1
_juneau.load()[0].plan(planned_steps(build_only, test_only, full, args.no_js_tests))
exit_code = 0
last_test_output = ""
global _js_skipped
_js_skipped = False
from juneau_terminal_scripts import traces
since = traces.mark()
if build_only or full:
exit_code, _ = build()
if exit_code != 0:
say("\nāŒ Build failed!", "error")
return exit_code
say("\nāœ… Build succeeded!")
if test_only or full:
if full:
say("\n" + "=" * 80)
test_start = time.time()
exit_code, last_test_output = test(no_container=args.no_container)
test_elapsed = time.time() - test_start
# The summarizer parsed the Reactor Summary already; parse it here only when it did not run (no juneau_terminal,
# or it failed).
module_times_, unmapped = _last_times or module_times(last_test_output, REPO_ROOT)
if exit_code != 0:
_, failures, errors = parse_test_results(last_test_output)
if failures is not None and errors is not None:
say(f"\nāŒ Java Tests failed! ({failures + errors} failed: {failures} failures, {errors} errors)", "error")
else:
say("\nāŒ Java Tests failed!", "error")
else:
say("\nāœ… Java Tests passed!")
if exit_code != 0:
return exit_code
perf_exit = run_perf_guard(test_elapsed, PERF_BASELINE, module_times_, enforce=args.enforce_perf, unmapped=unmapped)
if perf_exit != 0:
return perf_exit
py_exit = script_tests()
if py_exit != 0:
return py_exit
js_changed = [] if (args.js_tests or args.no_js_tests) else changed_files_vs_origin()
js_exit = maybe_run_js_tests(args.js_tests, args.no_js_tests, js_changed, installed=full)
if js_exit != 0:
return js_exit
full_run = not (args.no_container or args.no_js_tests or _js_skipped)
return run_trace_guard(since, args.warn_traces, full_run)
return exit_code
def _rebaseline_from_history(window, min_runs):
"""--rebaseline: rewrite perf-baseline.txt from this repo's test.py run history. Exit 1 with a message when the
history is off."""
run = _juneau.load()[0]
if not run.history_enabled():
print("āŒ run history is off (JUNEAU_TERMINAL_HISTORY=0), so there are no timings to rebaseline from; "
"nothing rewritten.")
return 1
return rebaseline(PERF_BASELINE, run.HistoryStore(REPO_ROOT, "test").read(), window=window, min_runs=min_runs)
def owned_by_enclosing_run():
"""True when push.py or an enclosing run spawned us; it rings once at its own end. The toolkit session makes the
same decision (juneau_terminal_scripts.run._nested_env), so this is the documented predicate rather than a branch here."""
return bool(os.environ.get("JUNEAU_PUSH_ACTIVE") or os.environ.get("JUNEAU_TERMINAL_N_BASE")
or os.environ.get("JUNEAU_TERMINAL_ACTIVE"))
def _prepare_output(argv):
"""Export --console, --full-log, --board-log and --detail before the session opens, so its console view sees
them (an argument beats a value already in the environment). _main parses and validates them again; an invalid
value is left for it to report."""
parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
for flag in ("--console", "--full-log", "--board-log", "--detail"):
parser.add_argument(flag)
args, _ = parser.parse_known_args(argv)
if args.console in ("full", "board", "none"):
os.environ["JUNEAU_TERMINAL_CONSOLE"] = args.console
for flag, variable in ((args.full_log, "JUNEAU_TERMINAL_FULL_LOG"), (args.board_log, "JUNEAU_TERMINAL_BOARD_LOG")):
if flag:
os.environ[variable] = flag
try:
_juneau.load()[0].export_detail(args.detail)
except ValueError:
pass
def main():
"""One toolkit session around _main: the run/done markers, the console view when one is requested, SIGTERM and
Ctrl-C as a cancel (exit 130), and the bell when a standalone run ends (never under push.py or an enclosing run;
JUNEAU_PUSH_NO_SOUND=1 silences it)."""
root = Path(__file__).resolve().parent.parent
_prepare_output(sys.argv[1:])
_juneau.load()
from juneau_terminal_scripts import run as toolkit
with toolkit.session("Test", header=["Juneau", git_value(['rev-parse', '--abbrev-ref', 'HEAD'])], bell=True, silence_env=("JUNEAU_PUSH_NO_SOUND",), repo=root, mode="test",
project=root.name, exit=False) as s:
code = _main()
if code:
raise SystemExit(code)
return s.code
if __name__ == '__main__':
sys.exit(main())