| #!/usr/bin/env python3 | |
| # *************************************************************************************************************************** | |
| # * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file | |
| # * distributed with this work for additional information regarding copyright ownership. The ASF licenses this file | |
| # * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance | |
| # * with the License. You may obtain a copy of the License at | |
| # * | |
| # * http://www.apache.org/licenses/LICENSE-2.0 | |
| # * | |
| # * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an | |
| # * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the | |
| # * specific language governing permissions and limitations under the License. | |
| # *************************************************************************************************************************** | |
| """ | |
| Builds Juneau, runs the Maven tests, then checks timings, runs the scripts' own pytest suite, and optionally the browser tests. | |
| Full docs: test.md | |
| """ | |
| USAGE = """ | |
| Build and test helper script for Apache Juneau. | |
| Usage: | |
| ./scripts/test.py [options] | |
| Options: | |
| --build-only, -b Only build (skip tests) | |
| --test-only, -t Only run tests (no build) | |
| --full, -f Clean build + run tests (default) | |
| --verbose, -v Accepted for compatibility; Maven output always streams live | |
| --no-container Exclude @Tag("container") tests | |
| --timing-log <path> Ignored, with a warning (timings are in the run history now); removed in a later release | |
| --enforce-perf Hard-fail on a perf-guard breach (see "Perf guard" below) | |
| --rebaseline Rewrite perf-baseline.txt's suite line and [modules] from this repo's test.py run | |
| history (~/.cache/juneau-terminal/history), then exit (no build) | |
| --window <n> --rebaseline: runs per key to take the median of (default 20); a module whose window | |
| spreads by more than 50% ((max - min) / min) takes the p75 instead (the suite keeps the median) | |
| --min-runs <n> --rebaseline: skip keys with fewer runs than this (default 5) | |
| --js-tests Also run the headless-browser JS harness (mvn -Pjs-tests, *_BrowserTest in | |
| juneau-console-views, juneau-petstore-core and juneau-terminal-region). Fails if Node/npm are missing. | |
| --no-js-tests Never run the JS harness (overrides auto-detect) | |
| --warn-traces Report stack traces in the test output as a warning instead of failing (see "Trace guard" below) | |
| --console <mode> Maven output on the console: full (default), board or none | |
| --detail <level> Console detail: summary, actionable (default), modules or all (JUNEAU_TERMINAL_DETAIL) | |
| --full-log <path> Also write every byte Maven prints to this file (appended) | |
| --board-log <path> Also write the board one-line-per-event stream to this file (appended) | |
| --profile <module> Run one-shot JFR profile for module tests | |
| --help, -h Show this help message | |
| Environment: | |
| JUNEAU_MVN_WRAPPER Optional prefix for every mvn command (e.g. a lock script that serializes | |
| concurrent runs on one checkout). | |
| Python Tests: | |
| After the Maven tests pass, scripts/tests (the tooling's own pytest suite) runs as its own row. The launcher is | |
| `python3 -m pytest` when pytest is importable, else `uv run --native-tls --with pytest --with pyyaml` when uv is on the PATH; with neither, | |
| the step prints a notice and skips. Nothing is installed. | |
| Browser Tests: | |
| The *_BrowserTest classes (juneau-console-views, juneau-petstore-core, juneau-terminal-region) are excluded from the main test | |
| pass, so it reports no skips for them; this step runs them with -Pjs-tests, as CI's js-tests and | |
| js-tests-petstore and js-tests-terminal jobs do. When neither | |
| --js-tests nor --no-js-tests is given, this script enables them automatically if any .js, .cjs, .css or .ftl | |
| file under a src/ tree differs from origin/master (committed, uncommitted or untracked). If Node/npm | |
| are not on the PATH, an auto-enabled run prints a notice and skips; an explicit --js-tests fails instead. | |
| The first run downloads Playwright + Chromium (needs network); later runs reuse target/js. | |
| Trace guard: | |
| After the other steps pass, a Traces row scans what they printed (read back from the run's full log) for stack | |
| traces, and fails on any whose header line is not matched by scripts/trace-allowlist.txt. The log exists with | |
| --console board or none; with the default --console full there is none, so the row is skipped with a warning. | |
| --warn-traces reports traces as a warning instead; an allowlist error still fails. The report of every trace, | |
| allowed or not, goes to target/trace-guard.txt. | |
| Perf guard: | |
| After the tests pass, the guard compares the measured wall clock of `mvn test` against 'suite' in the | |
| project-root perf-baseline.txt, and each module's reactor wall clock (from Maven's Reactor Summary, mapped to | |
| module paths by each pom's own <name>) against its [modules] entry. | |
| The suite breaches above baseline Ć (1 + JUNEAU_CI_PERF_THRESHOLD, default 0.20). A module breaches only when | |
| it is over that AND more than JUNEAU_CI_PERF_MIN_SECONDS (default 3) over its baseline. A module with no entry | |
| warns and never fails. The report goes to target/perf-guard.txt. | |
| Without --enforce-perf the check runs in warn-only mode (prints results, never exits non-zero for a perf | |
| breach). Each run's history record keeps the suite time and every module's reactor time (the reactor_s | |
| series), which --rebaseline turns into new baselines. | |
| """ | |
| import argparse | |
| import math | |
| import os | |
| import re | |
| import shlex | |
| import shutil | |
| import statistics | |
| import subprocess | |
| import sys | |
| import time | |
| import xml.etree.ElementTree as ET | |
| from datetime import datetime | |
| from pathlib import Path | |
| # The shared loader (_juneau.py) sits beside this script; appended, so no script here shadows an installed module. | |
| if str(Path(__file__).resolve().parent) not in sys.path: | |
| sys.path.append(str(Path(__file__).resolve().parent)) | |
| import _juneau # noqa: E402 | |
| REPO_ROOT = Path(__file__).resolve().parent.parent | |
| TRACE_ALLOWLIST = Path(__file__).resolve().parent / "trace-allowlist.txt" | |
| TRACE_REPORT = REPO_ROOT / "target" / "trace-guard.txt" | |
| _step_n = 0 | |
| def next_step_number(): | |
| """Top-level step numbers continue after JUNEAU_TERMINAL_N_BASE, which push.py sets for the steps it spawns us for.""" | |
| global _step_n | |
| _step_n += 1 | |
| return int(os.environ.get("JUNEAU_TERMINAL_N_BASE", "0")) + _step_n | |
| def say(text, level="info"): | |
| """Script status text: through juneau_terminal (a filtered note once a console view owns the screen).""" | |
| _juneau.load()[0].say(text, level) | |
| _SHELL_OPERATORS = {"&&", "||", ";", "|"} | |
| def _join_words(words): | |
| """shlex.join, except that a word that is exactly a shell chaining operator stays an operator.""" | |
| return " ".join(w if w in _SHELL_OPERATORS else shlex.quote(w) for w in words) | |
| def mvn_wrapper_prefix(): | |
| """JUNEAU_MVN_WRAPPER as a shell-safe command prefix, or "" when unset. | |
| The value is a command plus arguments, not a shell fragment: it is split like a shell would split it | |
| (shlex.split, so a pre-quoted `'/a b/lock.sh' --x` works) and re-joined with each word quoted (shlex.join), | |
| so a path with spaces or `$`/`;` is never interpreted; a word that is exactly `&&`, `||`, `;` or `|` stays an operator. An unquoted path with spaces is also accepted: the | |
| longest leading run of words that names an existing file is taken as the executable. A value that does not | |
| parse (unbalanced quote) is passed through unchanged.""" | |
| wrapper = os.environ.get("JUNEAU_MVN_WRAPPER", "").strip() | |
| if not wrapper: | |
| return "" | |
| # Candidate executables end at a whitespace boundary (or the end); the longest one that is a file wins. | |
| ends = [m.start() for m in re.finditer(r"\s+", wrapper)] + [len(wrapper)] | |
| for end in reversed(ends): | |
| head = wrapper[:end] | |
| if os.path.isfile(head): | |
| try: | |
| rest = shlex.split(wrapper[end:]) | |
| except ValueError: | |
| rest = wrapper[end:].split() | |
| return _join_words([head, *rest]) | |
| try: | |
| return _join_words(shlex.split(wrapper)) | |
| except ValueError: | |
| return wrapper | |
| def wrap_mvn_command(cmd): | |
| """cmd prefixed with the quoted JUNEAU_MVN_WRAPPER, when one is set.""" | |
| prefix = mvn_wrapper_prefix() | |
| return f"{prefix} {cmd}" if prefix else cmd | |
| def run_command(cmd, step_id=None, title=None, label=None, summarize=None): | |
| """Run a command, streaming its output live, and return exit code and full output. | |
| A command with a step_id runs as that step through juneau_terminal's run_tool whatever the output routing, so its row | |
| is recorded in the run history; with markers off and the default sinks its output is the same as a plain run's. | |
| Without a step_id (or without juneau_terminal) it runs under Popen, as it always has. summarize(text), when given, | |
| runs after the command exits while its row is still open (see _record_buckets); never on the Popen path. | |
| """ | |
| project_root = REPO_ROOT | |
| # Optional command prefix (e.g. a lock script that serializes concurrent mvn runs on one checkout). | |
| if cmd.startswith("mvn "): | |
| cmd = wrap_mvn_command(cmd) | |
| say(f"Running: {cmd}") | |
| say("-" * 80) # NOSONAR python:S1192 - the same banner rule as the separator in _main; a constant hides it | |
| run = _juneau.load()[0] | |
| if step_id: | |
| # Through a shell, like the default path, so the quoted JUNEAU_MVN_WRAPPER prefix keeps its meaning. | |
| result = run.run_tool(["/bin/sh", "-c", cmd], "maven", step_id, title or step_id, n=next_step_number(), | |
| cwd=str(project_root), capture=True, label=label, summarize=summarize) | |
| return result.exit, result.output | |
| # Echo each line as it arrives so a long or hung build is visible, and keep it for parse_test_results(). | |
| lines = [] | |
| with subprocess.Popen(cmd, shell=True, cwd=str(project_root), stdout=subprocess.PIPE, stderr=subprocess.STDOUT, | |
| text=True, errors="replace", bufsize=1) as proc: | |
| for line in proc.stdout: | |
| sys.stdout.write(line) | |
| sys.stdout.flush() | |
| lines.append(line) | |
| return proc.returncode, "".join(lines) | |
| def git_value(args): | |
| try: | |
| result = subprocess.run(["git", *args], cwd=str(Path(__file__).parent.parent), capture_output=True, text=True, check=True) | |
| return result.stdout.strip() | |
| except Exception: | |
| return "unknown" | |
| def parse_test_results(output): | |
| matches = list(re.finditer(r"\[ERROR\]\s+Tests run:\s+(\d+),\s+Failures:\s+(\d+),\s+Errors:\s+(\d+)", output)) | |
| if matches: | |
| match = matches[-1] | |
| total = int(match.group(1)) | |
| failures = int(match.group(2)) | |
| errors = int(match.group(3)) | |
| return total, failures, errors | |
| return None, None, None | |
| # āāā Per-module timing/perf subsystem āāāāāāāāāāāāāāāāāāāāāāāāāāāāāāāā | |
| # | |
| # Tests live in each module's own src/test/java and | |
| # report into that module's target/surefire-reports/. The helpers below discover | |
| # ALL such report dirs under the reactor (not just juneau-integration-tests's), attribute each | |
| # Surefire XML to its OWNING module (the parent of target/surefire-reports), and | |
| # bucket each test class. | |
| # | |
| # Module key = the module's path relative to the repo root (e.g. "juneau-integration-tests", | |
| # "juneau-core/juneau-junit5"; MAY contain '/'). | |
| # Bucket ā {core, container.springboot, container.jetty, container.tomcat}. | |
| # | |
| # juneau-integration-tests additionally splits its reports into surefire-reports/{core,container}/ | |
| # via two Surefire executions; the recursive XML scan below transparently handles | |
| # both that nested layout and the standard flat layout every migrated module uses. | |
| # Container annotation marker -> bucket. Checked against the test class source. | |
| CONTAINER_MARKERS = ( | |
| ("SpringbootTest", "container.springboot"), | |
| ("JettyMicroserviceTest", "container.jetty"), | |
| ("TomcatMicroserviceTest", "container.tomcat"), | |
| ) | |
| def classify_bucket(module_dir: Path, class_name: str) -> str: | |
| """Bucket a test class as core or a container.* flavor via source-annotation inspection.""" | |
| source = module_dir / "src" / "test" / "java" / Path("/".join(class_name.split("."))).with_suffix(".java") | |
| if source.exists(): | |
| try: | |
| content = source.read_text(encoding="utf-8") | |
| for marker, bucket in CONTAINER_MARKERS: | |
| if marker in content: | |
| return bucket | |
| return "core" | |
| except OSError: | |
| pass | |
| # Source unreadable (rare): fall back to a name-based heuristic, defaulting to core. | |
| lowered = class_name.lower() | |
| if "springboot" in lowered: | |
| return "container.springboot" | |
| if "tomcat" in lowered: | |
| return "container.tomcat" | |
| if "jetty" in lowered: | |
| return "container.jetty" | |
| return "core" | |
| def aggregate_module_reports(module_dir: Path, reports_dir: Path) -> dict: | |
| """Return {bucket: {'tests': int, 'seconds': float}} for one module's surefire-reports tree.""" | |
| buckets: dict = {} | |
| for xml_file in sorted(reports_dir.rglob("TEST-*.xml")): | |
| try: | |
| root = ET.parse(xml_file).getroot() | |
| except (ET.ParseError, OSError): | |
| continue | |
| bucket = classify_bucket(module_dir, root.attrib.get("name", "")) | |
| stats = buckets.setdefault(bucket, {"tests": 0, "seconds": 0.0}) | |
| stats["tests"] += int(root.attrib.get("tests", 0)) | |
| stats["seconds"] += float(root.attrib.get("time", 0.0)) | |
| return buckets | |
| def discover_module_stats(repo_root: Path) -> dict: | |
| """Map module-key -> {bucket -> {tests, seconds}} for every surefire-reports dir under the reactor.""" | |
| modules: dict = {} | |
| for reports_dir in sorted(repo_root.rglob("target/surefire-reports")): | |
| if not reports_dir.is_dir(): | |
| continue | |
| module_dir = reports_dir.parent.parent | |
| module_key = module_dir.relative_to(repo_root).as_posix() | |
| buckets = aggregate_module_reports(module_dir, reports_dir) | |
| if buckets: | |
| modules[module_key] = buckets | |
| return modules | |
| # A Reactor Summary row, read the way juneau_terminal's Maven parser reads it (MavenParser.SUMMARY_ROW in juneau_terminal.py). | |
| SECTION = re.compile(r"^\[([\w.-]+)\]$") | |
| REACTOR_SUMMARY = re.compile(r"^\[INFO\] Reactor Summary") | |
| SUMMARY_ROW = re.compile(r"^\[INFO\] (.+?)(?: \.+)? (SUCCESS|FAILURE|SKIPPED)(?: \[\s*([^\]]+?)\s*\])?\s*$") | |
| ANSI = re.compile(r"\x1b\[[0-9;]*m") | |
| POM_SKIP_DIRS = {"target", "node_modules", "src", "build-overlay"} | |
| def _summary_seconds(text): | |
| """'7.513 s' -> 7.513, '02:45 min' -> 165.0, '01:28 h' -> 5280.0 (as juneau_terminal's _seconds).""" | |
| text = text.strip() | |
| m = re.match(r"^(\d+):(\d+)\s*(min|h)?$", text) | |
| if m: | |
| a, b, unit = int(m.group(1)), int(m.group(2)), m.group(3) | |
| return float(a * 3600 + b * 60 if unit == "h" else a * 60 + b) | |
| m = re.match(r"^([\d.]+)\s*s?$", text) | |
| return float(m.group(1)) if m else 0.0 | |
| def reactor_times(output: str) -> dict: | |
| """{module name: seconds} from the Reactor Summary of one Maven run's output. Only SUCCESS rows count.""" | |
| times: dict = {} | |
| in_summary = False | |
| for raw in output.splitlines(): | |
| line = ANSI.sub("", raw) | |
| if REACTOR_SUMMARY.match(line): | |
| in_summary = True | |
| continue | |
| row = SUMMARY_ROW.match(line) if in_summary else None | |
| if row and row.group(2) == "SUCCESS" and row.group(3): | |
| times[row.group(1).strip()] = _summary_seconds(row.group(3)) | |
| return times | |
| def module_names(repo_root: Path) -> tuple: | |
| """({project name: module path}, [ambiguous names]) for every pom.xml outside target/, node_modules/, src/, | |
| build-overlay/ (helper poms packaged into a zip, not reactor modules) and dot-directories. | |
| The name is the project's own <name> (its <artifactId> when it has none), never its parent's; it is what the | |
| Reactor Summary prints. A name two poms share is left out and listed as ambiguous. The root pom is skipped. | |
| """ | |
| paths: dict = {} | |
| ambiguous: set = set() | |
| for dirpath, dirnames, filenames in os.walk(repo_root): | |
| dirnames[:] = sorted(d for d in dirnames if d not in POM_SKIP_DIRS and not d.startswith(".")) | |
| directory = Path(dirpath) | |
| if "pom.xml" not in filenames or directory == repo_root: | |
| continue | |
| try: | |
| root = ET.parse(directory / "pom.xml").getroot() | |
| except (ET.ParseError, OSError): | |
| continue | |
| ns = root.tag[:root.tag.index("}") + 1] if root.tag.startswith("{") else "" | |
| node = root.find(f"{ns}name") | |
| if node is None or not (node.text or "").strip(): | |
| node = root.find(f"{ns}artifactId") | |
| if node is None or not (node.text or "").strip(): | |
| continue | |
| name = node.text.strip() | |
| if name in paths or name in ambiguous: | |
| paths.pop(name, None) | |
| ambiguous.add(name) | |
| else: | |
| paths[name] = directory.relative_to(repo_root).as_posix() | |
| return paths, sorted(ambiguous) | |
| def module_times(output: str, repo_root: Path) -> tuple: | |
| """({module path: reactor seconds}, [names with no single pom]) for one `mvn test` run's output.""" | |
| reactor = reactor_times(output) | |
| if not reactor: | |
| return {}, [] | |
| names, _ = module_names(repo_root) | |
| times = {names[name]: seconds for name, seconds in reactor.items() if name in names} | |
| return times, sorted(name for name in reactor if name not in names) | |
| def read_baselines(baseline_file: Path) -> dict: | |
| """Parse perf-baseline.txt: 'suite' from anywhere, and every '<module path> = <seconds>' under [modules]. | |
| Entries are '<key> = <seconds>', optionally followed by a '#' comment. A '[name]' line starts a section. Blank | |
| lines, comments and every other section (e.g. [observability], which scripts/microbench.py reads) are ignored. | |
| """ | |
| result: dict = {} | |
| if not baseline_file.exists(): | |
| return result | |
| section = None | |
| for line in baseline_file.read_text(encoding="utf-8").splitlines(): | |
| stripped = line.split("#", 1)[0].strip() | |
| heading = SECTION.match(stripped) | |
| if heading: | |
| section = heading.group(1) | |
| continue | |
| if "=" not in stripped: | |
| continue | |
| key, _, value = stripped.partition("=") | |
| key = key.strip() | |
| if key != "suite" and section != "modules": | |
| continue | |
| try: | |
| result[key] = float(value.strip().split()[0]) | |
| except (ValueError, IndexError): | |
| continue | |
| return result | |
| PERF_REPORT = Path(__file__).resolve().parent.parent / "target" / "perf-guard.txt" | |
| PERF_BASELINE = Path(__file__).resolve().parent.parent / "perf-baseline.txt" | |
| class _PerfOutput: | |
| """Where PERF-GUARD lines go: notes on the Perf row when a console view owns the screen, else print. Every line | |
| also goes to the report file.""" | |
| def __init__(self, report_file): | |
| run = _juneau.load()[0] | |
| self.run = run if run.console_owns_screen() else None | |
| self.report_file = Path(report_file) | |
| self.lines = [] | |
| def line(self, text, level="info"): | |
| self.lines.append(text) | |
| if self.run is not None: | |
| self.run.note(level, text, step="perf") | |
| else: | |
| print(text) | |
| def report_only(self, text): | |
| self.lines.append(text) | |
| def write(self): | |
| self.report_file.parent.mkdir(parents=True, exist_ok=True) | |
| self.report_file.write_text("\n".join(self.lines) + "\n", encoding="utf-8") | |
| def run_perf_guard(test_elapsed: float, baseline_file: Path, module_times: dict, enforce: bool, report_file=None, | |
| unmapped=()) -> int: | |
| """Perf guard: the suite's wall clock, and each module's reactor wall clock (module_times, from module_times()). | |
| A breach is a warn note, or an error with enforce=True. Modules with no baseline collapse into one info note. | |
| The full report goes to report_file (target/perf-guard.txt); the Perf row shows the counts and its path. | |
| Returns exit code: 0 = pass or warn-only mode, 1 = threshold breached with enforce=True. | |
| """ | |
| run = _juneau.load()[0] | |
| with run.row("perf", "Perf", label="Perf") as row: | |
| out = _PerfOutput(report_file or PERF_REPORT) | |
| code, summary = _perf_guard(test_elapsed, baseline_file, module_times, unmapped, enforce, out) | |
| row.summary = summary | |
| if code != 0: | |
| row.status = "fail" | |
| return code | |
| def _perf_guard(test_elapsed, baseline_file, module_times, unmapped, enforce, out): | |
| """The check itself. Returns (exit code, Perf row summary).""" | |
| tolerance = float(os.environ.get("JUNEAU_CI_PERF_THRESHOLD", "0.20")) | |
| min_seconds = float(os.environ.get("JUNEAU_CI_PERF_MIN_SECONDS", "3")) | |
| breach = "error" if enforce else "warn" | |
| baselines = read_baselines(baseline_file) | |
| if not baselines: | |
| out.line(f"PERF-GUARD: baseline file not found or empty ({baseline_file}); skipping check.", "warn") | |
| out.write() | |
| return 0, f"no baseline file ā {out.report_file}" | |
| pct = f"±{tolerance * 100:.0f}%" | |
| over = 0 | |
| # 1) Suite-level guard. | |
| suite_baseline = baselines.get("suite") | |
| if suite_baseline is not None: | |
| threshold = suite_baseline * (1 + tolerance) | |
| if test_elapsed > threshold: | |
| out.line( | |
| f"PERF-GUARD FAIL [suite]: tests took {test_elapsed:.1f}s " | |
| f"(baseline {suite_baseline}s, tolerance {pct}, threshold {threshold:.1f}s).\n" | |
| f" If intentional, bump the 'suite' entry in perf-baseline.txt (project root).", breach) | |
| over += 1 | |
| else: | |
| out.line(f"PERF-GUARD OK [suite]: tests took {test_elapsed:.1f}s (baseline {suite_baseline}s, tolerance {pct}).") | |
| else: | |
| out.line("PERF-GUARD WARN [suite]: no 'suite' baseline configured; skipping wall-clock check.") | |
| # 2) Modules: over by the tolerance AND by min_seconds, so a short module's jitter never trips it. | |
| margins = f"{pct} and {min_seconds:g}s" | |
| checked = 0 | |
| regressions = 0 | |
| new_modules = [] | |
| for key in sorted(module_times): | |
| actual = module_times[key] | |
| baseline = baselines.get(key) | |
| if baseline is None: | |
| new_modules.append(key) | |
| continue | |
| checked += 1 | |
| threshold = max(baseline * (1 + tolerance), baseline + min_seconds) | |
| if actual > threshold: | |
| out.line( | |
| f"PERF-GUARD FAIL [{key}]: {actual:.1f}s " | |
| f"(baseline {baseline}s, tolerance {margins}, threshold {threshold:.1f}s).\n" | |
| f" If intentional, run `python3 scripts/test.py --rebaseline` or bump '{key}' in perf-baseline.txt " | |
| f"(project root).", breach) | |
| regressions += 1 | |
| over += 1 | |
| lines = [f"PERF-GUARD WARN [{key}]: {module_times[key]:.1f}s ā no baseline entry yet (new/unknown module; not failing)." | |
| for key in new_modules] | |
| if out.run is None: | |
| for text in lines: | |
| out.line(text) | |
| else: | |
| for text in lines: | |
| out.report_only(text) | |
| if new_modules: | |
| out.line(f"PERF-GUARD: {len(new_modules)} module(s) have no baseline entry yet: " | |
| + ", ".join(new_modules)) | |
| if unmapped: | |
| out.report_only(f"PERF-GUARD: {len(unmapped)} reactor module(s) not mapped to a pom (name shared or missing): " | |
| + ", ".join(unmapped)) | |
| out.line( | |
| f"PERF-GUARD SUMMARY: {checked} module(s) checked, {regressions} regression(s), " | |
| f"{len(new_modules)} new/unknown." | |
| ) | |
| if over and not enforce: | |
| out.line("PERF-GUARD: warn-only mode (pass --enforce-perf to hard-fail on breach).") | |
| out.write() | |
| if over: | |
| summary = f"ā {over} over baseline" + (f" Ā· {len(new_modules)} without one" if new_modules else "") | |
| else: | |
| summary = f"ā {checked} checked" + (f" Ā· {len(new_modules)} without a baseline" if new_modules else "") | |
| summary += f" ā {out.report_file}" | |
| return (1 if over and enforce else 0), summary | |
| SUITE_LINE = re.compile(r"^suite\s*=") | |
| OUTLIER_FACTOR = 3 | |
| BIMODAL_SPREAD = 0.5 | |
| def baseline_series(records) -> dict: | |
| """{key: [seconds...]} in run order over the passing `tests` records with the newest record's sig (its command | |
| shape): 'suite' from each record's wall_s, and a module path from each entry of its reactor_s series.""" | |
| series: dict = {} | |
| for record in _juneau.same_shape_runs(records): | |
| if record.get("ok") is not True: | |
| continue | |
| if isinstance(record.get("wall_s"), (int, float)): | |
| series.setdefault("suite", []).append(float(record["wall_s"])) | |
| reactor = (record.get("series") or {}).get("reactor_s") | |
| for key, seconds in (reactor.items() if isinstance(reactor, dict) else ()): | |
| if isinstance(seconds, (int, float)) and not isinstance(seconds, bool): | |
| series.setdefault(key, []).append(float(seconds)) | |
| return series | |
| def rebaseline_value(values: list, window: int, bimodal_p75: bool = False) -> float: | |
| """The median of the last `window` values, after dropping any over 3Ć their median, rounded up to 0.1s (a | |
| baseline is a ceiling). With `bimodal_p75`, a series whose kept values spread by more than BIMODAL_SPREAD | |
| ((max - min) / min) takes its 75th percentile instead of the median.""" | |
| recent = values[-window:] | |
| median = statistics.median(recent) | |
| kept = [v for v in recent if v <= OUTLIER_FACTOR * median] | |
| pick = statistics.median(kept) | |
| # A module that flips between a fast and a slow mode (CPU contention from neighbouring modules) has a median in | |
| # the fast mode, so the guard would fire on every slow-mode run; the p75 sits in the slow mode. | |
| if bimodal_p75 and len(kept) >= 2 and (max(kept) - min(kept)) / min(kept) > BIMODAL_SPREAD: | |
| pick = statistics.quantiles(kept, n=4, method="inclusive")[2] | |
| return math.ceil(round(pick * 10, 6)) / 10 | |
| def rebaseline(baseline_file: Path, records, window: int = 20, min_runs: int = 5) -> int: | |
| """Rewrite perf-baseline.txt's suite line and [modules] section from the history medians; print what changed. | |
| records are test.py's run-history records (HistoryStore(repo, "test").read()); baseline_series keeps the passing | |
| `tests` ones of the newest record's sig. When that command shape has fewer than min_runs passing runs, prints a | |
| "no baseline for this command shape" line and writes nothing. | |
| A key with fewer than min_runs passing runs in the window is skipped and keeps its value; a [modules] entry with | |
| no history at all is removed. Everything outside the suite line and the [modules] body is kept byte-for-byte. | |
| Runs no build. Returns 0, or 1 when there is no history or no [modules] line (nothing is written then). | |
| """ | |
| text = baseline_file.read_text(encoding="utf-8") if baseline_file.exists() else "" | |
| lines = text.splitlines(keepends=True) | |
| start = next((i for i, line in enumerate(lines) if line.strip() == "[modules]"), None) | |
| if start is None: | |
| print(f"ā {baseline_file} has no [modules] section line; nothing rewritten.") | |
| return 1 | |
| series = baseline_series(records) | |
| if not series: | |
| print("ā no passing runs in test.py's run history; nothing rewritten.") | |
| return 1 | |
| runs = sum(1 for r in _juneau.same_shape_runs(records) if r.get("ok") is True) | |
| if runs < min_runs: | |
| print(f"š {_juneau.no_baseline_line(runs)}; nothing rewritten.") | |
| return 1 | |
| end = next((i for i in range(start + 1, len(lines)) if SECTION.match(lines[i].strip())), len(lines)) | |
| # Comments and blank lines just above the next section (its banner) belong to it, not to [modules]. | |
| while end > start + 1 and (not lines[end - 1].strip() or lines[end - 1].lstrip().startswith("#")): | |
| end -= 1 | |
| before = read_baselines(baseline_file) | |
| fresh, skipped = {}, {} | |
| for key, values in series.items(): | |
| if len(values[-window:]) < min_runs: | |
| skipped[key] = len(values[-window:]) | |
| else: | |
| fresh[key] = rebaseline_value(values, window, bimodal_p75=key != "suite") | |
| modules = {k: v for k, v in before.items() if k != "suite" and k in skipped} | |
| modules.update({k: v for k, v in fresh.items() if k != "suite"}) | |
| if not lines[start].endswith("\n"): | |
| lines[start] += "\n" | |
| body = [f"{key} = {modules[key]:g}\n" for key in sorted(modules)] | |
| if end < len(lines) and lines[end].strip(): | |
| body.append("\n") | |
| out = lines[:start + 1] + body + lines[end:] | |
| if "suite" in fresh: | |
| suite_line = f"suite = {fresh['suite']:g}\n" | |
| at = next((i for i, line in enumerate(out) if SUITE_LINE.match(line)), None) | |
| if at is None: | |
| out.insert(start, suite_line) | |
| else: | |
| out[at] = suite_line | |
| baseline_file.write_text("".join(out), encoding="utf-8") | |
| after = read_baselines(baseline_file) | |
| keys = sorted(set(before) | set(after), key=lambda k: (k != "suite", k)) | |
| changed = [k for k in keys if k in before and k in after and before[k] != after[k]] | |
| added = [k for k in keys if k not in before] | |
| removed = [k for k in keys if k not in after] | |
| print(f"š Rebaselined {baseline_file} from the run history (last {window} passing runs): {len(changed)} changed, " | |
| f"{len(added)} added, {len(removed)} removed, {len(skipped)} skipped (fewer than {min_runs} runs).") | |
| for k in changed: | |
| print(f" ~ {k}: {before[k]:g} ā {after[k]:g}") | |
| for k in added: | |
| print(f" + {k}: {after[k]:g}") | |
| for k in removed: | |
| print(f" - {k}: {before[k]:g}") | |
| for k in sorted(skipped): | |
| print(f" ? {k}: {skipped[k]} run(s)") | |
| return 0 | |
| # Reactor-level parallel *module* builds. -T1C runs one build thread per CPU core, so independent | |
| # reactor modules (and their test forks) build concurrently, overlapping the otherwise-serial | |
| # per-module test phases. Requires the juneau-distrib dependency:copy ordering fix (MDEP-187) to be | |
| # -T-safe. This is parallel *module* execution only, NOT in-JVM concurrent test classes/methods. | |
| PARALLELISM = "-T1C" | |
| def build(): | |
| return run_command(f"mvn clean install {PARALLELISM} -DskipTests", "build", "Build", label="Compile") | |
| _last_times = None # (module times, unmapped names) of the last test() run, from _record_buckets | |
| def _record_buckets(text): | |
| """run_tool's summarize for the tests step, called while its row is open: one surefire_s.<bucket> series per bucket | |
| with data ({module path: summed Surefire class seconds}), and reactor_s ({module path: reactor seconds}, the perf | |
| guard's metric, also kept in _last_times for _main). Returns None, so the row keeps the parser's summary.""" | |
| global _last_times | |
| run = _juneau.load()[0] | |
| per_bucket = {} | |
| for module, buckets in discover_module_stats(REPO_ROOT).items(): | |
| for bucket, stats in buckets.items(): | |
| per_bucket.setdefault(bucket, {})[module] = round(stats["seconds"], 3) | |
| for bucket, values in sorted(per_bucket.items()): | |
| run.history_series(f"surefire_s.{bucket}", values) | |
| _last_times = module_times(text or "", REPO_ROOT) | |
| if _last_times[0]: | |
| run.history_series("reactor_s", _last_times[0]) | |
| return None | |
| def test(no_container=False): | |
| global _last_times | |
| _last_times = None | |
| cmd = f"mvn test {PARALLELISM} -Drat.skip=true" | |
| if no_container: | |
| cmd += " -DexcludedGroups=container" | |
| return run_command(cmd, "tests", "Java Tests", label="Java Tests", summarize=_record_buckets) | |
| # Set in the pytest child so a test that reaches script_tests() again cannot launch the suite recursively. | |
| SCRIPT_TESTS_ACTIVE = "JUNEAU_SCRIPT_TESTS_ACTIVE" | |
| def script_tests_launcher(): | |
| """The command prefix that runs pytest, or None. `python3 -m pytest` when pytest is importable, else | |
| `uv run --native-tls --with pytest --with pyyaml` when uv is on the PATH. Nothing is ever installed.""" | |
| python = shutil.which("python3") or sys.executable | |
| try: | |
| importable = subprocess.run([python, "-c", "import pytest"], capture_output=True).returncode == 0 | |
| except OSError: | |
| importable = False | |
| if importable: | |
| return [python, "-m", "pytest"] | |
| if shutil.which("uv"): | |
| return ["uv", "run", "--native-tls", "--with", "pytest", "--with", "pyyaml", "pytest"] | |
| return None | |
| def script_tests_command(launcher): | |
| return [*launcher, "-q", "-p", "no:cacheprovider", "scripts/tests"] | |
| def script_tests(launcher=None, runner=None): | |
| """The scripts/tests pytest suite as its own row. Returns 0 on pass or skip, non-zero on failure.""" | |
| if os.environ.get(SCRIPT_TESTS_ACTIVE): | |
| say("\nā ļø Skipping Python Tests: already running inside them (no recursion).", "warn") | |
| skip_py_tests_row("already running inside them") | |
| return 0 | |
| launcher = launcher or script_tests_launcher() | |
| if launcher is None: | |
| say("\nā ļø Skipping Python Tests: neither pytest nor uv found (python3 -m pytest / uv run --native-tls --with pytest --with pyyaml).", "warn") | |
| skip_py_tests_row("neither pytest nor uv found") | |
| return 0 | |
| cmd = script_tests_command(launcher) | |
| say("\nš Running Python Tests (scripts/tests)...") | |
| say(f"Running: {' '.join(cmd)}") | |
| say("-" * 80) # NOSONAR python:S1192 - the same banner rule as run_command | |
| env = dict(os.environ, PYTHONDONTWRITEBYTECODE="1", **{SCRIPT_TESTS_ACTIVE: "1"}) | |
| if runner is not None: | |
| code = runner(cmd, env) | |
| else: | |
| run = _juneau.load()[0] | |
| code = run.run_tool(cmd, "pytest", "py-tests", "Python Tests", n=next_step_number(), cwd=str(REPO_ROOT), | |
| env=env, capture=True, label="Python Tests").exit | |
| say("\nā Python Tests passed!" if code == 0 else "\nā Python Tests failed!", "info" if code == 0 else "error") | |
| return code | |
| # The modules whose *_BrowserTest classes the browser step runs, with the package-qualified -Dtest pattern that | |
| # selects each module's own suite (the same patterns as CI's js-tests, js-tests-petstore and js-tests-terminal jobs; a bare | |
| # '*_BrowserTest' would also match a sibling module's suite in the reactor). Those classes are excluded from the | |
| # default surefire pass by each module's pom and enabled by its js-tests profile. | |
| JS_TEST_MODULES = ( | |
| ("juneau-rest/juneau-console-views", "org.apache.juneau.console.views.*_BrowserTest"), | |
| ("juneau-petstore/juneau-petstore-core", "org.apache.juneau.petstore.**.*_BrowserTest"), | |
| ("juneau-rest/juneau-terminal-region", "org.apache.juneau.terminal.region.*_BrowserTest"), | |
| ) | |
| JS_SOURCE_SUFFIXES = (".js", ".cjs", ".css", ".ftl") | |
| def is_js_source(path): | |
| """True for a .js/.css/.ftl file that lives under a src/ tree (any module).""" | |
| p = path.replace("\\", "/") | |
| parts = p.split("/") | |
| return p.lower().endswith(JS_SOURCE_SUFFIXES) and "src" in parts[:-1] | |
| def changed_files_vs_origin(repo_root=None, base="origin/master"): | |
| """Paths changed in HEAD, the working tree or untracked, relative to `base`. Empty list if git/base unavailable.""" | |
| repo_root = repo_root or Path(__file__).parent.parent | |
| names = [] | |
| try: | |
| for args in (["diff", "--name-only", base], ["ls-files", "--others", "--exclude-standard"]): | |
| r = subprocess.run(["git", *args], cwd=str(repo_root), capture_output=True, text=True, check=True) | |
| names.extend(line for line in r.stdout.splitlines() if line.strip()) | |
| except Exception: | |
| return [] | |
| return names | |
| def should_run_js_tests(js_flag, no_js_flag, changed_files): | |
| """Resolve (enabled, explicit). Explicit flags win (--no-js-tests over --js-tests); otherwise auto-detect.""" | |
| if no_js_flag: | |
| return False, False | |
| if js_flag: | |
| return True, True | |
| return any(is_js_source(f) for f in changed_files), False | |
| def js_prereq_problem(): | |
| """Return a description of the missing browser prerequisite, or None if Node and npm are on the PATH.""" | |
| missing = [tool for tool in ("node", "npm") if shutil.which(tool) is None] | |
| if missing: | |
| return f"{' and '.join(missing)} not found on the PATH (the -Pjs-tests profile needs Node and npm)" | |
| return None | |
| def js_tests_command(): | |
| """One reactor run over every JS test module. .mvn/maven.config's --also-make builds the upstream modules from | |
| the working tree, so the same command serves a run after this script's install and a --test-only run.""" | |
| modules = ",".join(module for module, _ in JS_TEST_MODULES) | |
| patterns = ",".join(pattern for _, pattern in JS_TEST_MODULES) | |
| return (f"mvn -Pjs-tests -pl {modules} test -Drat.skip=true " | |
| f"-Dtest='{patterns}' -Dsurefire.failIfNoSpecifiedTests=false") | |
| # Forks per module while the modules overlap; a lone petstore run takes the pom's default of 3. | |
| JS_CONCURRENT_FORKS = 2 | |
| def js_tests_concurrent_command(): | |
| """The same suites as js_tests_command, as one mvn per module so they overlap (one reactor runs them in dependency | |
| order). Each `mvn -f` resolves its upstream modules from ~/.m2 (hence -o), so this is valid only straight after the | |
| install step. Output streams live with a [module] prefix; the shell waits for every module and fails if any did, | |
| and a module that left no exit status counts as a failure. Needs bash (PIPESTATUS).""" | |
| runs = [] | |
| for i, (module, pattern) in enumerate(JS_TEST_MODULES): | |
| runs.append(f"(mvn -o -Pjs-tests -f {module}/pom.xml test -Drat.skip=true -Dtest='{pattern}' " | |
| f"-Dsurefire.failIfNoSpecifiedTests=false -Djuneau.jsTests.forks={JS_CONCURRENT_FORKS} 2>&1 " | |
| f"| sed -u 's|^|[{module}] |' | tee target/js-tests-{i}.log; echo ${{PIPESTATUS[0]}} > target/js-tests-{i}.exit) &") | |
| codes = " ".join(f"$(cat target/js-tests-{i}.exit 2>/dev/null || echo 1)" for i in range(len(JS_TEST_MODULES))) | |
| return ("mkdir -p target && rm -f target/js-tests-*.exit && " + " ".join(runs) | |
| + f" wait ; for c in {codes}; do [ \"$c\" = 0 ] || exit 1; done") | |
| def js_tests(installed=False): | |
| """The JS harness: the views, petstore and terminal browser suites. Straight after the install step the three | |
| modules run at once (js_tests_concurrent_command); otherwise one sequential reactor run (js_tests_command), since | |
| the upstream modules are then built from the working tree rather than read from ~/.m2.""" | |
| if not installed: | |
| return run_command(js_tests_command(), "js-tests", "Browser Tests", label="Browser Tests") | |
| cmd = f"/bin/bash -c {shlex.quote(js_tests_concurrent_command())}" | |
| # Wraps the whole compound: run_command only prefixes commands that start with "mvn ". | |
| cmd = wrap_mvn_command(cmd) | |
| return run_command(cmd, "js-tests", "Browser Tests", label="Browser Tests") | |
| #: test.py's top-level steps in run order, as (id, label). The labels are push.py's PLAN_LABELS for the same ids: | |
| #: under push.py this script's pending rows stand in for the parent's. | |
| PLAN = (("build", "Compile"), ("tests", "Java Tests"), ("perf", "Perf"), ("py-tests", "Python Tests"), | |
| ("js-tests", "Browser Tests"), ("traces", "Traces")) | |
| def planned_steps(build_only, test_only, full, no_js_tests): | |
| """The steps this run will take, in order, as (id, label) pairs, for the console's pending rows and Overall | |
| figure. --no-js-tests leaves Browser Tests out; a run-time skip (no JS changes) stays in the plan, and its skip | |
| row is left out of the count.""" | |
| ids = {"build"} if build_only or full else set() | |
| if test_only or full: | |
| ids |= {"tests", "perf", "py-tests", "traces"} | (set() if no_js_tests else {"js-tests"}) | |
| return [(step_id, label) for step_id, label in PLAN if step_id in ids] | |
| _js_skipped = False # the Browser Tests row was skipped this run, so JS-only allowlist entries cannot have matched | |
| def run_trace_guard(since, warn_only, full_run): | |
| """The Traces row: the output of the steps since `since` (traces.mark(), taken before the build) must hold no stack | |
| trace that is not in scripts/trace-allowlist.txt. warn_only (--warn-traces) downgrades a failure to a warning. | |
| full_run lets the guard report allowlist entries that matched nothing. Returns 0 or 1.""" | |
| from juneau_terminal_scripts import traces | |
| return traces.gate("traces", "Traces", TRACE_ALLOWLIST, since=since, warn_only=warn_only, full_run=full_run, | |
| report=TRACE_REPORT, label="Traces") | |
| def skip_py_tests_row(reason): | |
| """Draw the Python Tests row as skipped, with the reason as its text.""" | |
| with _juneau.load()[0].row("py-tests", "Python Tests", label="Python Tests") as row: | |
| row.skip(reason) | |
| def skip_js_tests_row(reason): | |
| """Draw the Browser Tests row as skipped, with the reason as its text.""" | |
| global _js_skipped | |
| _js_skipped = True | |
| with _juneau.load()[0].row("js-tests", "Browser Tests", label="Browser Tests") as row: | |
| row.skip(reason) | |
| def maybe_run_js_tests(js_flag, no_js_flag, changed_files, runner=None, installed=False): | |
| """Run the JS harness if enabled. Returns 0 on pass/skip, non-zero on failure. A skipped harness still gets its | |
| row, marked skipped with the reason.""" | |
| enabled, explicit = should_run_js_tests(js_flag, no_js_flag, changed_files) | |
| if not enabled: | |
| skip_js_tests_row("--no-js-tests" if no_js_flag | |
| else "no JS changes since origin/master (--js-tests to force)") | |
| return 0 | |
| problem = js_prereq_problem() | |
| if problem: | |
| if explicit: | |
| say(f"\nā --js-tests requested but cannot run: {problem}.", "error") | |
| return 1 | |
| say(f"\nā ļø JS files changed, but skipping Browser Tests: {problem}. (CI will still run them.)", "warn") | |
| skip_js_tests_row(problem.split(" (")[0]) | |
| return 0 | |
| say("\nš Running Browser Tests (-Pjs-tests)..." + ("" if explicit else " (auto: JS/CSS/FTL files changed)")) | |
| code, _ = (runner or (lambda: js_tests(installed)))() | |
| say("\nā Browser Tests passed!" if code == 0 else "\nā Browser Tests failed!", "info" if code == 0 else "error") | |
| return code | |
| def module_surefire_extra_args(module_dir): | |
| """The module pom's own <surefire.extraArgs> value (e.g. the mockito agent), or "" when it sets none.""" | |
| match = re.search(r"<surefire\.extraArgs>(.*?)</surefire\.extraArgs>", (module_dir / "pom.xml").read_text(), re.S) | |
| return match.group(1).strip() if match else "" | |
| def profile_command(module, output_path): | |
| """The mvn command that profiles one module, run from that module's directory. | |
| The jacoco agent is skipped, because its prepare-agent goal rewrites argLine after any -DargLine, and the JFR flags | |
| ride on -Dsurefire.extraArgs (which the surefire argLine already appends) after the module's own value. | |
| """ | |
| module_dir = REPO_ROOT / module | |
| jfr = f"-XX:FlightRecorderOptions:stackdepth=1024 -XX:StartFlightRecording=filename={output_path},settings=profile,dumponexit=true" | |
| extra = " ".join(part for part in (module_surefire_extra_args(module_dir), jfr) if part) | |
| mvn = f"mvn test -Drat.skip=true -Djacoco.skip=true {shlex.quote('-Dsurefire.extraArgs=' + extra)}" | |
| return f"cd {shlex.quote(str(module_dir))} && {wrap_mvn_command(mvn)}" | |
| def profile(module): | |
| ts = datetime.now().strftime("%Y%m%d-%H%M%S") | |
| profile_dir = (REPO_ROOT / "target" / "profile-results").resolve() | |
| profile_dir.mkdir(parents=True, exist_ok=True) | |
| safe_module = module.replace("/", "-") | |
| # An absolute directory, so it does not resolve against each forked JVM's working directory. JFR writes one | |
| # hotspot-pid-<pid>-id-<n>-<timestamp>.jfr into it per JVM, so executions and forks never overwrite each other. | |
| output_dir = profile_dir / f"{safe_module}-{ts}" | |
| output_dir.mkdir() | |
| code, out = run_command(profile_command(module, output_dir), "profile", "Profile", label="Profile") | |
| if code != 0: | |
| return code, out | |
| written = sorted(output_dir.glob("*.jfr")) | |
| if not written: | |
| say(f"\nā mvn passed but no JFR file was written under {output_dir}.", "error") | |
| return 1, out | |
| for jfr_file in written: | |
| say(f"\nā JFR profile captured at {jfr_file}", "warn") | |
| return code, out | |
| def _main(): # NOSONAR python:S3776 -- Cognitive complexity is acceptable for this main function | |
| parser = argparse.ArgumentParser(add_help=False) | |
| parser.add_argument("--build-only", "-b", action="store_true") | |
| parser.add_argument("--test-only", "-t", action="store_true") | |
| parser.add_argument("--full", "-f", action="store_true") | |
| parser.add_argument("--verbose", "-v", action="store_true") | |
| parser.add_argument("--no-container", action="store_true") | |
| parser.add_argument("--timing-log") | |
| parser.add_argument("--enforce-perf", action="store_true") | |
| parser.add_argument("--rebaseline", action="store_true") | |
| parser.add_argument("--window", type=int, default=20) | |
| parser.add_argument("--min-runs", type=int, default=5, dest="min_runs") | |
| parser.add_argument("--js-tests", action="store_true", dest="js_tests") | |
| parser.add_argument("--no-js-tests", action="store_true", dest="no_js_tests") | |
| parser.add_argument("--warn-traces", action="store_true", dest="warn_traces") | |
| parser.add_argument("--profile") | |
| parser.add_argument("--console", choices=("full", "board", "none")) | |
| parser.add_argument("--detail") | |
| parser.add_argument("--full-log") | |
| parser.add_argument("--board-log") | |
| parser.add_argument("--help", "-h", action="store_true") | |
| args, unknown = parser.parse_known_args() | |
| if args.help: | |
| print(USAGE) | |
| return 0 | |
| if unknown: | |
| print(f"Unknown option(s): {' '.join(unknown)}") | |
| print(USAGE) | |
| return 1 | |
| if args.timing_log: | |
| say("ā --timing-log is ignored: timings are in the run history now (scripts/push-timings.py reads it). " | |
| "The option goes away in a later release.", "warn") | |
| if args.rebaseline: | |
| return _rebaseline_from_history(args.window, args.min_runs) | |
| # juneau_terminal reads its output routing from the environment; an argument beats a value already set there. | |
| for flag, variable in ((args.console, "JUNEAU_TERMINAL_CONSOLE"), (args.full_log, "JUNEAU_TERMINAL_FULL_LOG"), | |
| (args.board_log, "JUNEAU_TERMINAL_BOARD_LOG")): | |
| if flag: | |
| os.environ[variable] = flag | |
| try: | |
| _juneau.load()[0].export_detail(args.detail) | |
| except ValueError as e: | |
| print(e) | |
| return 2 | |
| build_only = args.build_only | |
| test_only = args.test_only | |
| full = args.full or not (build_only or test_only or args.profile) | |
| if args.profile: | |
| exit_code, _ = profile(args.profile) | |
| return exit_code | |
| if build_only and test_only: | |
| print("Cannot combine --build-only and --test-only") | |
| return 1 | |
| if args.js_tests and args.no_js_tests: | |
| print("Cannot combine --js-tests and --no-js-tests") | |
| return 1 | |
| _juneau.load()[0].plan(planned_steps(build_only, test_only, full, args.no_js_tests)) | |
| exit_code = 0 | |
| last_test_output = "" | |
| global _js_skipped | |
| _js_skipped = False | |
| from juneau_terminal_scripts import traces | |
| since = traces.mark() | |
| if build_only or full: | |
| exit_code, _ = build() | |
| if exit_code != 0: | |
| say("\nā Build failed!", "error") | |
| return exit_code | |
| say("\nā Build succeeded!") | |
| if test_only or full: | |
| if full: | |
| say("\n" + "=" * 80) | |
| test_start = time.time() | |
| exit_code, last_test_output = test(no_container=args.no_container) | |
| test_elapsed = time.time() - test_start | |
| # The summarizer parsed the Reactor Summary already; parse it here only when it did not run (no juneau_terminal, | |
| # or it failed). | |
| module_times_, unmapped = _last_times or module_times(last_test_output, REPO_ROOT) | |
| if exit_code != 0: | |
| _, failures, errors = parse_test_results(last_test_output) | |
| if failures is not None and errors is not None: | |
| say(f"\nā Java Tests failed! ({failures + errors} failed: {failures} failures, {errors} errors)", "error") | |
| else: | |
| say("\nā Java Tests failed!", "error") | |
| else: | |
| say("\nā Java Tests passed!") | |
| if exit_code != 0: | |
| return exit_code | |
| perf_exit = run_perf_guard(test_elapsed, PERF_BASELINE, module_times_, enforce=args.enforce_perf, unmapped=unmapped) | |
| if perf_exit != 0: | |
| return perf_exit | |
| py_exit = script_tests() | |
| if py_exit != 0: | |
| return py_exit | |
| js_changed = [] if (args.js_tests or args.no_js_tests) else changed_files_vs_origin() | |
| js_exit = maybe_run_js_tests(args.js_tests, args.no_js_tests, js_changed, installed=full) | |
| if js_exit != 0: | |
| return js_exit | |
| full_run = not (args.no_container or args.no_js_tests or _js_skipped) | |
| return run_trace_guard(since, args.warn_traces, full_run) | |
| return exit_code | |
| def _rebaseline_from_history(window, min_runs): | |
| """--rebaseline: rewrite perf-baseline.txt from this repo's test.py run history. Exit 1 with a message when the | |
| history is off.""" | |
| run = _juneau.load()[0] | |
| if not run.history_enabled(): | |
| print("ā run history is off (JUNEAU_TERMINAL_HISTORY=0), so there are no timings to rebaseline from; " | |
| "nothing rewritten.") | |
| return 1 | |
| return rebaseline(PERF_BASELINE, run.HistoryStore(REPO_ROOT, "test").read(), window=window, min_runs=min_runs) | |
| def owned_by_enclosing_run(): | |
| """True when push.py or an enclosing run spawned us; it rings once at its own end. The toolkit session makes the | |
| same decision (juneau_terminal_scripts.run._nested_env), so this is the documented predicate rather than a branch here.""" | |
| return bool(os.environ.get("JUNEAU_PUSH_ACTIVE") or os.environ.get("JUNEAU_TERMINAL_N_BASE") | |
| or os.environ.get("JUNEAU_TERMINAL_ACTIVE")) | |
| def _prepare_output(argv): | |
| """Export --console, --full-log, --board-log and --detail before the session opens, so its console view sees | |
| them (an argument beats a value already in the environment). _main parses and validates them again; an invalid | |
| value is left for it to report.""" | |
| parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False) | |
| for flag in ("--console", "--full-log", "--board-log", "--detail"): | |
| parser.add_argument(flag) | |
| args, _ = parser.parse_known_args(argv) | |
| if args.console in ("full", "board", "none"): | |
| os.environ["JUNEAU_TERMINAL_CONSOLE"] = args.console | |
| for flag, variable in ((args.full_log, "JUNEAU_TERMINAL_FULL_LOG"), (args.board_log, "JUNEAU_TERMINAL_BOARD_LOG")): | |
| if flag: | |
| os.environ[variable] = flag | |
| try: | |
| _juneau.load()[0].export_detail(args.detail) | |
| except ValueError: | |
| pass | |
| def main(): | |
| """One toolkit session around _main: the run/done markers, the console view when one is requested, SIGTERM and | |
| Ctrl-C as a cancel (exit 130), and the bell when a standalone run ends (never under push.py or an enclosing run; | |
| JUNEAU_PUSH_NO_SOUND=1 silences it).""" | |
| root = Path(__file__).resolve().parent.parent | |
| _prepare_output(sys.argv[1:]) | |
| _juneau.load() | |
| from juneau_terminal_scripts import run as toolkit | |
| with toolkit.session("Test", header=["Juneau", git_value(['rev-parse', '--abbrev-ref', 'HEAD'])], bell=True, silence_env=("JUNEAU_PUSH_NO_SOUND",), repo=root, mode="test", | |
| project=root.name, exit=False) as s: | |
| code = _main() | |
| if code: | |
| raise SystemExit(code) | |
| return s.code | |
| if __name__ == '__main__': | |
| sys.exit(main()) |