| #!/usr/bin/env python3 | |
| # *************************************************************************************************************************** | |
| # * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file * | |
| # * distributed with this work for additional information regarding copyright ownership. The ASF licenses this file * | |
| # * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance * | |
| # * with the License. You may obtain a copy of the License at * | |
| # * * | |
| # * http://www.apache.org/licenses/LICENSE-2.0 * | |
| # * * | |
| # * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an * | |
| # * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * | |
| # * specific language governing permissions and limitations under the License. * | |
| # *************************************************************************************************************************** | |
| """ | |
| Regenerate the generated images of the docs site, or check them. | |
| Full docs: docs-images.md | |
| """ | |
| import argparse | |
| import fnmatch | |
| import html | |
| import json | |
| import os | |
| import re | |
| import shlex | |
| import shutil | |
| import signal | |
| import socket | |
| import subprocess | |
| import sys | |
| import tempfile | |
| import threading | |
| import time | |
| import unicodedata | |
| import urllib.parse | |
| import urllib.request | |
| from pathlib import Path | |
| from juneau_root import find_juneau_root | |
| CASES_DIR_NAME = 'docs-images' | |
| SOURCES = {'ui': 'ui-cases.json', 'terminal': 'terminal-cases.json'} | |
| IMAGE_DIRS = ('topics', 'developer-info') | |
| HEX_COLOR = '#%02x%02x%02x' | |
| IMAGE_EXTS = ('.png', '.svg', '.jpg', '.jpeg', '.gif', '.webp') | |
| DEFAULT_PORT = 10000 | |
| PROBE_PATH = '/console/store' | |
| DEFAULT_VIEWPORT = [1280, 900] | |
| DEFAULT_SCALE = 2 | |
| SERVER_START_TIMEOUT = 600 | |
| # The petstore reads this to pin its clock to the seed data's own, so the store page's day figures repeat run to run. | |
| FIXED_NOW_ENV = 'PETSTORE_FIXED_NOW' | |
| FIXED_NOW = 'seed' | |
| # A site-absolute image path: /topics/... or /developer-info/..., not part of a longer path such as /docs/topics/... | |
| IMAGE_REF_RE = re.compile(r'(?<![\w/.-])/((?:' + '|'.join(re.escape(d) for d in IMAGE_DIRS) + r')/[^\s)"\'#?]+?\.(?:' | |
| + '|'.join(e[1:] for e in IMAGE_EXTS) + r'))(?=[\s)"\'#?]|$)', re.IGNORECASE) | |
| PETSTORE_CORE = Path('juneau-petstore') / 'juneau-petstore-core' | |
| HARNESS = PETSTORE_CORE / 'src' / 'test' / 'js' / 'petstore-browser.cjs' | |
| class Problem(Exception): | |
| """A failure reported to the user without a traceback.""" | |
| # --------------------------------------------------------------------------------------------------------------------- | |
| # Cases files | |
| # --------------------------------------------------------------------------------------------------------------------- | |
| def load_cases(cases_dir, sources=None): | |
| """ | |
| Load the cases files. | |
| Returns: | |
| (cases, unmanaged): cases is a list of dicts, each with its "source" added and the file's "defaults" merged | |
| in; unmanaged is the list of exempt globs from every file loaded. | |
| """ | |
| cases, unmanaged, seen = [], [], {} | |
| for source, file_name in SOURCES.items(): | |
| if sources and source not in sources: | |
| continue | |
| path = Path(cases_dir) / file_name | |
| if not path.is_file(): | |
| continue | |
| data = json.loads(path.read_text(encoding='utf-8')) | |
| defaults = data.get('defaults', {}) | |
| for entry in data.get('unmanaged', []): | |
| unmanaged.append(entry['glob'] if isinstance(entry, dict) else entry) | |
| for raw in data.get('cases', []): | |
| case = {**defaults, **raw, 'source': source} | |
| for key in ('id', 'image'): | |
| if not case.get(key): | |
| raise Problem(f"{path.name}: a case has no '{key}': {json.dumps(raw)}") | |
| if case['id'] in seen: | |
| raise Problem(f"{path.name}: duplicate case id '{case['id']}'") | |
| seen[case['id']] = case | |
| cases.append(case) | |
| images = {} | |
| for case in cases: | |
| if case['image'] in images: | |
| raise Problem(f"cases '{images[case['image']]}' and '{case['id']}' both write {case['image']}") | |
| images[case['image']] = case['id'] | |
| return cases, unmanaged | |
| def select_cases(cases, only): | |
| """The cases whose id or image matches one of the --only patterns (globs), or all of them.""" | |
| if not only: | |
| return list(cases) | |
| picked = [c for c in cases if any(fnmatch.fnmatchcase(c['id'], p) or fnmatch.fnmatchcase(c['image'], p) | |
| or fnmatch.fnmatchcase(Path(c['image']).name, p) for p in only)] | |
| if not picked: | |
| raise Problem(f"--only {' '.join(only)} matches no case") | |
| return picked | |
| # --------------------------------------------------------------------------------------------------------------------- | |
| # --check | |
| # --------------------------------------------------------------------------------------------------------------------- | |
| def page_references(pages_dir): | |
| """Map of each referenced image (path relative to static/) to the 'file:line' places that reference it.""" | |
| refs = {} | |
| for md in sorted(Path(pages_dir).rglob('*.md*')): | |
| for n, line in enumerate(md.read_text(encoding='utf-8').splitlines(), 1): | |
| for m in IMAGE_REF_RE.finditer(line): | |
| refs.setdefault(m.group(1), []).append(f"{md.relative_to(pages_dir.parent)}:{n}") | |
| return refs | |
| def image_files(static_dir): | |
| """Every image file under the managed static directories, relative to static/ with forward slashes.""" | |
| out = [] | |
| for d in IMAGE_DIRS: | |
| root = Path(static_dir) / d | |
| if root.is_dir(): | |
| out += [p.relative_to(static_dir).as_posix() for p in root.rglob('*') | |
| if p.is_file() and p.suffix.lower() in IMAGE_EXTS] | |
| return sorted(out) | |
| def check(docs_root, cases, unmanaged): | |
| """ | |
| Returns: | |
| (errors, warnings): lists of "KIND path: detail" lines. | |
| """ | |
| static_dir, pages_dir = Path(docs_root) / 'static', Path(docs_root) / 'pages' | |
| refs = page_references(pages_dir) | |
| produced = {c['image']: c for c in cases} | |
| errors, warnings = [], [] | |
| for image, case in produced.items(): | |
| if not (static_dir / image).is_file(): | |
| errors.append(f"MISSING static/{image}: case '{case['id']}' has not been generated") | |
| for image in image_files(static_dir): | |
| if image not in produced and not any(fnmatch.fnmatchcase(image, g) for g in unmanaged): | |
| errors.append(f"ORPHANED static/{image}: no case produces it and no 'unmanaged' glob exempts it") | |
| for image, places in sorted(refs.items()): | |
| if not (static_dir / image).is_file(): | |
| errors.extend(f"DANGLING {place}: /{image} does not exist" for place in places) | |
| for image, case in produced.items(): | |
| if image not in refs: | |
| warnings.append(f"UNREFERENCED static/{image}: no page references case '{case['id']}'") | |
| return errors, warnings | |
| # --------------------------------------------------------------------------------------------------------------------- | |
| # UI source | |
| # --------------------------------------------------------------------------------------------------------------------- | |
| def shot_name(case): | |
| """The file the harness writes for a case, in the scratch directory.""" | |
| return case['id'] + Path(case['image']).suffix | |
| def harness_input(cases, base_url, snapshots): | |
| """Translate UI cases into the petstore-browser.cjs input.""" | |
| out = [] | |
| for c in cases: | |
| shot = {'screenshot': shot_name(c)} | |
| if c.get('selector'): | |
| shot['selector'] = c['selector'] | |
| hc = {'name': c['id'], 'path': c['path'], 'viewport': c.get('viewport', DEFAULT_VIEWPORT), | |
| 'deviceScaleFactor': c.get('deviceScaleFactor', DEFAULT_SCALE), | |
| 'actions': list(c.get('actions', [])) + [shot]} | |
| for key in ('waitUntil', 'initScript', 'allowFailedLoads'): | |
| if key in c: | |
| hc[key] = c[key] | |
| out.append(hc) | |
| return {'baseUrl': base_url.rstrip('/'), 'snapshots': str(snapshots), 'cases': out} | |
| def case_failures(cases, report, snapshots): | |
| """'id: reason' for every case that did not produce a clean screenshot.""" | |
| failures = [] | |
| for c in cases: | |
| r = report.get(c['id']) | |
| if r is None: | |
| failures.append(f"{c['id']}: no report from the harness") | |
| continue | |
| why = [] | |
| if r.get('status') != 200: | |
| why.append(f"HTTP status {r.get('status')}") | |
| if r.get('timeouts'): | |
| why.append('timed out waiting for ' + '; '.join(r['timeouts'])) | |
| if r.get('jsFailures'): | |
| why.append('script errors: ' + '; '.join(r['jsFailures'])) | |
| if not (Path(snapshots) / shot_name(c)).is_file(): | |
| why.append('no screenshot written') | |
| if why: | |
| failures.append(f"{c['id']}: " + ', '.join(why)) | |
| return failures | |
| def playwright_version(juneau_root): | |
| pom = (Path(juneau_root) / PETSTORE_CORE / 'pom.xml').read_text(encoding='utf-8') | |
| m = re.search(r'<playwright\.version>([^<]+)</playwright\.version>', pom) | |
| if not m: | |
| raise Problem(f"no <playwright.version> in {PETSTORE_CORE / 'pom.xml'}") | |
| return m.group(1) | |
| def ensure_playwright(juneau_root): | |
| """The js directory holding node_modules/playwright and the Chromium download, installing them if missing.""" | |
| for tool in ('node', 'npm'): | |
| if not shutil.which(tool): | |
| raise Problem(f"'{tool}' is not on the PATH; the ui source needs Node.js") | |
| js_dir = Path(juneau_root) / PETSTORE_CORE / 'target' / 'js' | |
| env = {**os.environ, 'PLAYWRIGHT_BROWSERS_PATH': str(js_dir / 'browsers')} | |
| cli = js_dir / 'node_modules' / 'playwright' / 'cli.js' | |
| if not cli.is_file(): | |
| version = playwright_version(juneau_root) | |
| print(f"-> Installing playwright@{version} into {js_dir}") | |
| run_checked(['npm', 'install', '--no-fund', '--no-audit', '--prefix', str(js_dir), f'playwright@{version}']) | |
| if not any((js_dir / 'browsers').glob('chromium*')): | |
| print(f"-> Installing Chromium into {js_dir / 'browsers'}") | |
| run_checked(['node', str(cli), 'install', 'chromium'], env=env) | |
| return js_dir | |
| def run_checked(cmd, **kwargs): | |
| r = subprocess.run(cmd, capture_output=True, text=True, **kwargs) | |
| if r.returncode != 0: | |
| tail = '\n'.join((r.stdout + r.stderr).splitlines()[-20:]) | |
| raise Problem(f"command failed ({r.returncode}): {' '.join(cmd)}\n{tail}") | |
| return r | |
| def generate_ui(docs_root, juneau_root, cases, base_url): | |
| js_dir = ensure_playwright(juneau_root) | |
| harness = Path(juneau_root) / HARNESS | |
| if not harness.is_file(): | |
| raise Problem(f"harness not found: {harness}") | |
| with tempfile.TemporaryDirectory(prefix='docs-images-') as tmp: | |
| snapshots = Path(tmp) / 'snapshots' | |
| snapshots.mkdir() | |
| input_file = Path(tmp) / 'input.json' | |
| input_file.write_text(json.dumps(harness_input(cases, base_url, snapshots), indent=1), encoding='utf-8') | |
| env = {**os.environ, 'NODE_PATH': str(js_dir / 'node_modules'), | |
| 'PLAYWRIGHT_BROWSERS_PATH': str(js_dir / 'browsers')} | |
| print(f"-> Shooting {len(cases)} UI case(s) against {base_url}") | |
| r = run_checked(['node', str(harness), str(input_file)], env=env) | |
| report = json.loads(r.stdout).get('cases', {}) | |
| failures = case_failures(cases, report, snapshots) | |
| if failures: | |
| raise Problem('UI cases failed, no image was replaced:\n ' + '\n '.join(failures)) | |
| for c in cases: | |
| target = Path(docs_root) / 'static' / c['image'] | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| shutil.move(str(snapshots / shot_name(c)), str(target)) | |
| print(f" static/{c['image']}") | |
| # --------------------------------------------------------------------------------------------------------------------- | |
| # Terminal source | |
| # --------------------------------------------------------------------------------------------------------------------- | |
| TERM_DEFAULT_COLUMNS, TERM_DEFAULT_ROWS, TERM_DEFAULT_TIMEOUT = 100, 30, 60 | |
| JUNEAU_TERMINAL_PY = Path('juneau-terminal') / 'src' / 'main' / 'python' / 'juneau_terminal.py' | |
| FIXTURES_DIR = Path('scripts') / CASES_DIR_NAME / 'fixtures' | |
| # The font stack of the site's code blocks (Infima's default); the SVG's text is real text in this font. | |
| TERM_FONT = 'SFMono-Regular, Menlo, Monaco, Consolas, "Liberation Mono", "Courier New", monospace' | |
| TERM_FONT_SIZE, TERM_CELL_W, TERM_LINE_H, TERM_PAD = 14, 8.4, 18, 12 | |
| TERM_FG, TERM_BG, TERM_DIVIDER = '#d4d4d4', '#1e1e1e', '#3c3c3c' | |
| # The 16 ANSI colours (VS Code dark), one fixed palette for both site themes: a terminal looks like a terminal. | |
| TERM_PALETTE = ['#000000', '#cd3131', '#0dbc79', '#e5e510', '#2472c8', '#bc3fbc', '#11a8cd', '#e5e5e5', | |
| '#666666', '#f14c4c', '#23d18b', '#f5f543', '#3b8eea', '#d670d6', '#29b8db', '#ffffff'] | |
| ANSI_CSI_RE = re.compile(r'\x1b\[([0-9;:?<=>!"$#%&\' ]*)([@-~])') | |
| ANSI_OTHER_RE = re.compile(r'\x1b(?:\][^\x07\x1b]*(?:\x07|\x1b\\)|[()][0-9A-Za-z]|[@-Z\\-_=>])') | |
| # default style: (fg, bg, bold, dim, underline, reverse); fg and bg are None or a '#rrggbb' string | |
| DEFAULT_STYLE = (None, None, False, False, False, False) | |
| def color_256(n): | |
| """The '#rrggbb' of xterm colour n.""" | |
| if n < 16: | |
| return TERM_PALETTE[n] | |
| if n < 232: | |
| n -= 16 | |
| return HEX_COLOR % tuple(0 if c == 0 else 55 + 40 * c for c in (n // 36, n // 6 % 6, n % 6)) | |
| v = 8 + 10 * (n - 232) | |
| return HEX_COLOR % (v, v, v) | |
| def apply_sgr(style, params): | |
| """The style after one SGR sequence; unknown codes are ignored.""" | |
| fg, bg, bold, dim, underline, reverse = style | |
| codes = [int(p) if p.isdigit() else 0 for p in (params.replace(':', ';') or '0').split(';')] | |
| i = 0 | |
| while i < len(codes): | |
| c = codes[i] | |
| if c == 0: | |
| fg, bg, bold, dim, underline, reverse = DEFAULT_STYLE | |
| elif c == 1: | |
| bold = True | |
| elif c == 2: | |
| dim = True | |
| elif c == 4: | |
| underline = True | |
| elif c == 7: | |
| reverse = True | |
| elif c == 22: | |
| bold = dim = False | |
| elif c == 24: | |
| underline = False | |
| elif c == 27: | |
| reverse = False | |
| elif 30 <= c <= 37: | |
| fg = TERM_PALETTE[c - 30] | |
| elif c == 39: | |
| fg = None | |
| elif 40 <= c <= 47: | |
| bg = TERM_PALETTE[c - 40] | |
| elif c == 49: | |
| bg = None | |
| elif 90 <= c <= 97: | |
| fg = TERM_PALETTE[c - 90 + 8] | |
| elif 100 <= c <= 107: | |
| bg = TERM_PALETTE[c - 100 + 8] | |
| elif c in (38, 48): | |
| color = None | |
| if i + 2 < len(codes) and codes[i + 1] == 5: | |
| color, i = color_256(min(codes[i + 2], 255)), i + 2 | |
| elif i + 4 < len(codes) and codes[i + 1] == 2: | |
| color, i = HEX_COLOR % tuple(min(v, 255) for v in codes[i + 2:i + 5]), i + 4 | |
| if color: | |
| fg, bg = (color, bg) if c == 38 else (fg, color) | |
| i += 1 | |
| return (fg, bg, bold, dim, underline, reverse) | |
| def char_width(ch): | |
| if unicodedata.combining(ch): | |
| return 0 | |
| return 2 if unicodedata.east_asian_width(ch) in ('W', 'F') else 1 | |
| def parse_ansi_line(line): | |
| """ | |
| One captured line as a list of cells [text, width, style]. SGR sequences set the style; every other escape | |
| sequence and control character is dropped, never emitted. | |
| """ | |
| cells, style, pos = [], DEFAULT_STYLE, 0 | |
| while pos < len(line): | |
| if line[pos] == '\x1b': | |
| m = ANSI_CSI_RE.match(line, pos) | |
| if m: | |
| if m.group(2) == 'm': | |
| style = apply_sgr(style, m.group(1)) | |
| pos = m.end() | |
| continue | |
| m = ANSI_OTHER_RE.match(line, pos) | |
| pos = m.end() if m else pos + 1 | |
| continue | |
| ch = line[pos] | |
| pos += 1 | |
| if ch < ' ' or ch == '\x7f': | |
| continue | |
| w = char_width(ch) | |
| if w == 0: | |
| if cells: | |
| cells[-1][0] += ch | |
| continue | |
| cells.append([ch, w, style]) | |
| while cells and cells[-1][0] == ' ' and cells[-1][2] == DEFAULT_STYLE: | |
| cells.pop() | |
| return cells | |
| def parse_ansi(text): | |
| """Captured pane text as a list of rows of cells.""" | |
| return [parse_ansi_line(line) for line in text.split('\n')] | |
| def style_colors(style): | |
| """(fg, bg) of a style as '#rrggbb', with reverse applied.""" | |
| fg, bg = style[0] or TERM_FG, style[1] or TERM_BG | |
| return (bg, fg) if style[5] else (fg, bg) | |
| def row_runs(cells): | |
| """The row's cells merged into runs of one style: (first column, columns, text, style).""" | |
| runs, col = [], 0 | |
| for text, width, style in cells: | |
| if runs and runs[-1][3] == style: | |
| runs[-1][1] += width | |
| runs[-1][2] += text | |
| else: | |
| runs.append([col, width, text, style]) | |
| col += width | |
| return runs | |
| def num(v): | |
| return ('%.2f' % v).rstrip('0').rstrip('.') | |
| def render_svg(panes, title, description): | |
| """ | |
| Render one or more panes side by side as an SVG whose text is real <text>: one <text> per row, one <tspan> per | |
| same-style run, each placed at its column. A pane is a list of rows of cells, as parse_ansi returns; a tuple of | |
| panes is a column of panes stacked top to bottom, one row apart. | |
| """ | |
| columns = [p if isinstance(p, tuple) else (p,) for p in panes] | |
| gap = 1 if len(columns) > 1 else 0 | |
| cols = [max([sum(c[1] for c in row) for rows in column for row in rows] + [1]) for column in columns] | |
| heights = [sum(len(rows) for rows in column) + len(column) - 1 for column in columns] | |
| total_cols = sum(cols) + gap * (len(columns) - 1) | |
| n_rows = max(max(heights), 1) | |
| width, height = total_cols * TERM_CELL_W + 2 * TERM_PAD, n_rows * TERM_LINE_H + 2 * TERM_PAD | |
| bg_parts, text_parts, x_col = [], [], 0 | |
| for p, column in enumerate(columns): | |
| if p: | |
| bg_parts.append(f'<rect x="{num(TERM_PAD + (x_col - gap) * TERM_CELL_W)}" y="{num(TERM_PAD)}" ' | |
| f'width="{num(gap * TERM_CELL_W)}" height="{num(n_rows * TERM_LINE_H)}" fill="{TERM_DIVIDER}"/>') | |
| y_row = 0 | |
| for q, rows in enumerate(column): | |
| if q: | |
| bg_parts.append(f'<rect x="{num(TERM_PAD + x_col * TERM_CELL_W)}" y="{num(TERM_PAD + y_row * TERM_LINE_H)}" ' | |
| f'width="{num(cols[p] * TERM_CELL_W)}" height="{TERM_LINE_H}" fill="{TERM_DIVIDER}"/>') | |
| y_row += 1 | |
| for r, cells in enumerate(rows): | |
| y = TERM_PAD + (y_row + r) * TERM_LINE_H | |
| spans = [] | |
| for col, w, text, style in row_runs(cells): | |
| fg, bg = style_colors(style) | |
| x = TERM_PAD + (x_col + col) * TERM_CELL_W | |
| if bg != TERM_BG: | |
| bg_parts.append(f'<rect x="{num(x)}" y="{num(y)}" width="{num(w * TERM_CELL_W)}" ' | |
| f'height="{TERM_LINE_H}" fill="{bg}"/>') | |
| # Words are placed one by one, so a run's inner spaces never stretch; an underlined run stays whole. | |
| segments = [(col, w, text)] if style[4] else [ | |
| (col + sum(map(char_width, text[:m.start()])), sum(map(char_width, m.group())), m.group()) | |
| for m in re.finditer(r'[^ ]+', text)] | |
| for seg_col, seg_w, seg_text in segments: | |
| attrs = f'x="{num(TERM_PAD + (x_col + seg_col) * TERM_CELL_W)}" ' \ | |
| f'textLength="{num(seg_w * TERM_CELL_W)}" fill="{fg}"' | |
| if style[2]: | |
| attrs += ' font-weight="bold"' | |
| if style[3]: | |
| attrs += ' fill-opacity="0.6"' | |
| if style[4]: | |
| attrs += ' text-decoration="underline"' | |
| spans.append(f'<tspan {attrs}>{html.escape(seg_text, quote=True)}</tspan>') | |
| if spans: | |
| text_parts.append(f'<text y="{num(y + TERM_LINE_H * 0.75)}">' + ''.join(spans) + '</text>') | |
| y_row += len(rows) | |
| x_col += cols[p] + gap | |
| return '\n'.join([ | |
| f'<svg xmlns="http://www.w3.org/2000/svg" width="{num(width)}" height="{num(height)}" ' | |
| f'viewBox="0 0 {num(width)} {num(height)}" role="img" aria-labelledby="title desc">', | |
| f'<title id="title">{html.escape(title, quote=True)}</title>', | |
| f'<desc id="desc">{html.escape(description, quote=True)}</desc>', | |
| f'<rect width="{num(width)}" height="{num(height)}" rx="6" fill="{TERM_BG}"/>', | |
| *bg_parts, | |
| f'<g font-family=\'{TERM_FONT}\' font-size="{TERM_FONT_SIZE}" xml:space="preserve">', | |
| *text_parts, | |
| '</g>', | |
| '</svg>', | |
| '']) | |
| def mask_text(text, masks): | |
| """Apply the case's [regex, replacement] pairs, line by line, to the captured text.""" | |
| for pattern, replacement in masks: | |
| text = re.sub(pattern, replacement, text, flags=re.MULTILINE) | |
| return text | |
| def tmux(socket_name, *args): | |
| return subprocess.run(['tmux', '-u', '-L', socket_name, '-f', '/dev/null', *args], capture_output=True, text=True) | |
| def expand_command(command, juneau_root, docs_root): | |
| subs = {'{juneau_terminal}': str(Path(juneau_root) / JUNEAU_TERMINAL_PY), '{fixtures}': str(Path(docs_root) / FIXTURES_DIR)} | |
| out = [] | |
| for arg in command: | |
| for key, value in subs.items(): | |
| arg = arg.replace(key, value) | |
| out.append(arg) | |
| return out | |
| def capture_pane(pane, juneau_root, docs_root, cwd): | |
| """ | |
| Run one pane spec in a detached tmux session of fixed size, wait until its "waitFor" regex shows and the pane has | |
| stopped changing, and return the captured text with escapes (tmux capture-pane -e -p), masked. The session lives | |
| on a private tmux server (own socket, no user config) that is killed afterwards. | |
| """ | |
| columns, rows = pane.get('columns', TERM_DEFAULT_COLUMNS), pane.get('rows', TERM_DEFAULT_ROWS) | |
| timeout = pane.get('timeout', TERM_DEFAULT_TIMEOUT) | |
| masks = pane.get('mask', []) | |
| wait_for = re.compile(pane['waitFor'], re.MULTILINE) if pane.get('waitFor') else None | |
| command = expand_command(pane['command'], juneau_root, docs_root) | |
| env = {'LANG': 'en_US.UTF-8', 'LC_ALL': 'en_US.UTF-8', **pane.get('env', {})} | |
| shell = 'env ' + ' '.join(shlex.quote(f'{k}={v}') for k, v in env.items()) + ' ' + shlex.join(command) + '; sleep 600' | |
| name = re.sub(r'[^\w-]', '_', f"docs-images-{os.getpid()}-{pane.get('id', 'pane')}") | |
| r = tmux(name, 'new-session', '-d', '-s', name, '-x', str(columns), '-y', str(rows), '-c', str(cwd), shell) | |
| if r.returncode != 0: | |
| raise Problem(f"tmux could not start the fixture: {r.stderr.strip()}") | |
| try: | |
| deadline = time.monotonic() + timeout | |
| last = None | |
| while True: | |
| plain = tmux(name, 'capture-pane', '-p', '-t', name).stdout | |
| if wait_for is None or wait_for.search(plain): | |
| shown = tmux(name, 'capture-pane', '-e', '-p', '-t', name).stdout | |
| masked = mask_text(shown, masks) | |
| if masked == last: | |
| return masked | |
| last = masked | |
| elif time.monotonic() > deadline: | |
| raise Problem(f"timed out after {timeout}s waiting for /{pane['waitFor']}/ in:\n{plain}") | |
| if time.monotonic() > deadline: | |
| raise Problem(f"the pane kept changing for {timeout}s; mask what varies:\n{last}") | |
| time.sleep(0.3) | |
| finally: | |
| tmux(name, 'kill-session', '-t', name) | |
| tmux(name, 'kill-server') | |
| # kill-server leaves the socket file behind | |
| (Path(os.environ.get('TMUX_TMPDIR', '/tmp')) / f'tmux-{os.getuid()}' / name).unlink(missing_ok=True) | |
| def trim_rows(rows): | |
| """The rows without the blank ones at the end (at least one row is kept).""" | |
| rows = list(rows) | |
| while len(rows) > 1 and not rows[-1]: | |
| rows.pop() | |
| return rows | |
| def render_terminal_case(case, juneau_root, docs_root): | |
| """The SVG text of one terminal case (a single pane, or its "panes" side by side; a {"stack": [...]} entry is a column).""" | |
| specs = case.get('panes') or [case] | |
| with tempfile.TemporaryDirectory(prefix='docs-images-term-') as cwd: | |
| def grid(spec, name): | |
| return trim_rows(parse_ansi(capture_pane({**spec, 'id': f"{case['id']}-{name}"}, juneau_root, docs_root, cwd))) | |
| grids = [tuple(grid(s, f'{i}-{j}') for j, s in enumerate(spec['stack'])) if 'stack' in spec else grid(spec, i) | |
| for i, spec in enumerate(specs)] | |
| return render_svg(grids, case.get('title', case['id']), case.get('description', '')) | |
| def generate_terminal(docs_root, juneau_root, cases): | |
| if not shutil.which('tmux'): | |
| raise Problem("'tmux' is not on the PATH; the terminal source needs tmux") | |
| if not (Path(juneau_root) / JUNEAU_TERMINAL_PY).is_file(): | |
| raise Problem(f"juneau_terminal.py not found: {Path(juneau_root) / JUNEAU_TERMINAL_PY}") | |
| print(f"-> Capturing {len(cases)} terminal case(s)") | |
| rendered = {} | |
| for c in cases: | |
| rendered[c['id']] = render_terminal_case(c, juneau_root, docs_root) | |
| for c in cases: | |
| target = Path(docs_root) / 'static' / c['image'] | |
| target.parent.mkdir(parents=True, exist_ok=True) | |
| target.write_text(rendered[c['id']], encoding='utf-8') | |
| print(f" static/{c['image']}") | |
| # --------------------------------------------------------------------------------------------------------------------- | |
| # Petstore server | |
| # --------------------------------------------------------------------------------------------------------------------- | |
| def looks_like_petstore(base_url, timeout=5): | |
| try: | |
| with urllib.request.urlopen(base_url.rstrip('/') + PROBE_PATH, timeout=timeout) as resp: | |
| return resp.status == 200 | |
| except (OSError, ValueError): # URLError is an OSError | |
| return False | |
| def select_for_server(cases, fresh): | |
| """Drops the fresh-server-only cases when the petstore was not started by this script, and warns about drift.""" | |
| if fresh: | |
| return cases | |
| print(f"WARNING: this petstore was not started by this script, so it does not run on the fixed clock " | |
| f"({FIXED_NOW_ENV}): shots that show the date or the time of day may differ from run to run.") | |
| kept = [c for c in cases if not c.get('freshServerOnly')] | |
| for c in cases: | |
| if c not in kept: | |
| print(f" skipping {c['id']}: it needs a petstore started by this script (it counts requests since start)") | |
| return kept | |
| class Server: | |
| """A petstore started by this script: it is ready once the console answers at url, and its process group is stopped.""" | |
| def __init__(self, cmd, cwd, url, timeout=SERVER_START_TIMEOUT, poll=2, env=None): | |
| self.cmd, self.url, self.lines = cmd, url, [] | |
| self.proc = subprocess.Popen(cmd, cwd=cwd, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE, | |
| stderr=subprocess.STDOUT, text=True, start_new_session=True, env=env) | |
| def pump(): | |
| for line in self.proc.stdout: | |
| self.lines.append(line.rstrip('\n')) | |
| threading.Thread(target=pump, daemon=True).start() | |
| deadline = time.monotonic() + timeout | |
| try: | |
| while not looks_like_petstore(url): | |
| if self.proc.poll() is not None or time.monotonic() > deadline: | |
| self.stop() | |
| time.sleep(0.2) # let the pump drain the last lines | |
| tail = '\n'.join(self.lines[-20:]) | |
| raise Problem(f"petstore did not start: {' '.join(cmd)}\n{tail}") | |
| time.sleep(poll) | |
| except BaseException: | |
| self.stop() | |
| raise | |
| def stop(self): | |
| if self.proc.poll() is not None: | |
| return | |
| try: | |
| os.killpg(self.proc.pid, signal.SIGTERM) | |
| self.proc.wait(30) | |
| except subprocess.TimeoutExpired: | |
| os.killpg(self.proc.pid, signal.SIGKILL) | |
| self.proc.wait() | |
| except ProcessLookupError: | |
| pass | |
| def port_holder(port): | |
| """A description of the process listening on port ('PID 123 (java)'), or None if it can't be found.""" | |
| try: | |
| r = subprocess.run(['lsof', '-nP', f'-iTCP:{port}', '-sTCP:LISTEN', '-Fpc'], | |
| capture_output=True, text=True, timeout=10) | |
| except (OSError, subprocess.SubprocessError): | |
| return None | |
| pid = cmd = None | |
| for line in r.stdout.splitlines(): | |
| if line.startswith('p') and pid is None: | |
| pid = line[1:] | |
| elif line.startswith('c') and cmd is None: | |
| cmd = line[1:] | |
| if not pid: | |
| return None | |
| suffix = f' ({cmd})' if cmd else '' | |
| return f'PID {pid}{suffix}' | |
| def port_in_use(url): | |
| parsed = urllib.parse.urlsplit(url) | |
| try: | |
| with socket.create_connection((parsed.hostname, parsed.port), timeout=2): | |
| return True | |
| except OSError: | |
| return False | |
| # --------------------------------------------------------------------------------------------------------------------- | |
| # Main | |
| # --------------------------------------------------------------------------------------------------------------------- | |
| def parse_args(argv): | |
| script_dir = Path(__file__).resolve().parent | |
| p = argparse.ArgumentParser(description='Regenerate or check the generated images of the docs site.') | |
| p.add_argument('--check', action='store_true', help='write nothing; report problems and exit 1 on any') | |
| p.add_argument('--only', nargs='+', metavar='PATTERN', help='case ids or image names (globs) to regenerate') | |
| p.add_argument('--source', choices=sorted(SOURCES), help='regenerate one source only') | |
| p.add_argument('--port', type=int, default=DEFAULT_PORT, metavar='N', | |
| help=f'port of the petstore to reuse or start (default {DEFAULT_PORT}); ignored with --base-url') | |
| p.add_argument('--base-url', help='an already-running petstore to shoot (default: reuse or start one)') | |
| p.add_argument('--juneau-root', type=Path, default=None, | |
| help='Juneau checkout (default: $JUNEAU_ROOT, then sibling ../master, then ../juneau)') | |
| p.add_argument('--docs-root', type=Path, default=script_dir.parent, help=argparse.SUPPRESS) | |
| return p.parse_args(argv) | |
| def main(argv=None): | |
| args = parse_args(argv) | |
| docs_root = args.docs_root.resolve() | |
| cases_dir = docs_root / 'scripts' / CASES_DIR_NAME | |
| try: | |
| if args.check: | |
| cases, unmanaged = load_cases(cases_dir) | |
| errors, warnings = check(docs_root, cases, unmanaged) | |
| for line in errors + warnings: | |
| print(line) | |
| print(f"{len(cases)} generated image(s): {len(errors)} problem(s), {len(warnings)} warning(s)") | |
| return 1 if errors else 0 | |
| cases, _ = load_cases(cases_dir, [args.source] if args.source else None) | |
| cases = select_cases(cases, args.only) | |
| ui = [c for c in cases if c['source'] == 'ui'] | |
| terminal = [c for c in cases if c['source'] == 'terminal'] | |
| if not ui and not terminal: | |
| print('Nothing to generate.') | |
| return 0 | |
| juneau_root = find_juneau_root(docs_root, args.juneau_root) | |
| if terminal: | |
| start = time.monotonic() | |
| generate_terminal(docs_root, juneau_root, terminal) | |
| print(f"Generated {len(terminal)} terminal image(s) in {time.monotonic() - start:.0f}s") | |
| if not ui: | |
| return 0 | |
| server = None | |
| base_url = args.base_url | |
| fresh = False | |
| if not base_url: | |
| default_url = f'http://localhost:{args.port}' | |
| if looks_like_petstore(default_url): | |
| base_url = default_url | |
| print(f"-> Using the petstore already running at {base_url}") | |
| elif port_in_use(default_url): | |
| holder = port_holder(args.port) | |
| raise Problem(f"port {args.port} is in use by something that is not the petstore" | |
| f"{f' ({holder})' if holder else ''}; stop it, or pass " | |
| "--port/--base-url for a petstore running elsewhere") | |
| else: | |
| print('-> Starting petstore-jetty (this can take a few minutes on a first build)') | |
| server = Server([sys.executable, str(juneau_root / 'scripts' / 'start-petstore-jetty.py'), | |
| '--port', str(args.port)], | |
| cwd=juneau_root, url=default_url, env={**os.environ, FIXED_NOW_ENV: FIXED_NOW}) | |
| base_url = server.url | |
| fresh = True | |
| print(f" started at {base_url}") | |
| elif not looks_like_petstore(base_url): | |
| raise Problem(f"{base_url}{PROBE_PATH} does not answer 200; is the petstore running there?") | |
| ui = select_for_server(ui, fresh) | |
| if not ui: | |
| print('Nothing to generate.') | |
| return 0 | |
| try: | |
| start = time.monotonic() | |
| generate_ui(docs_root, juneau_root, ui, base_url) | |
| print(f"Generated {len(ui)} image(s) in {time.monotonic() - start:.0f}s") | |
| finally: | |
| if server: | |
| server.stop() | |
| return 0 | |
| except Problem as e: | |
| print(f"ERROR: {e}", file=sys.stderr) | |
| return 1 | |
| except KeyboardInterrupt: | |
| print('Interrupted; no image was replaced.', file=sys.stderr) | |
| return 130 | |
| if __name__ == '__main__': | |
| sys.exit(main()) |