blob: 8c3c60dd078f6115cfce9ecb1bedb6ef320b680f [file]
#!/usr/bin/env python3
# ***************************************************************************************************************************
# * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file *
# * distributed with this work for additional information regarding copyright ownership. The ASF licenses this file *
# * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance *
# * with the License. You may obtain a copy of the License at *
# * *
# * http://www.apache.org/licenses/LICENSE-2.0 *
# * *
# * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an *
# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the *
# * specific language governing permissions and limitations under the License. *
# ***************************************************************************************************************************
"""
Gate, commit, push and stage the documentation: the juneau-docs counterpart of Juneau's push.py.
Full docs: push.md
"""
import argparse
import importlib.util
import os
import subprocess
import sys
import traceback
from pathlib import Path
import terminal_view
from juneau_root import find_juneau_root
_HELP_EPILOG = """
Push the documentation
This script:
1. Runs the Prechecks: git identity, branch docs, no unreviewed (unstaged or untracked) files, not behind upstream
2. Regenerates stale AI artifacts (static/llms.txt, static/ai/) and stages exactly the files that run rewrote
3. Runs build-docs.py (the one list of checks, plus the Docusaurus build); --staging unless --no-stage
Stages the javadoc pages that build rewrote (static/javadocs/) so they ride in the same commit
4. Commits whatever is staged (it stages nothing else) and pushes docs to its upstream
5. Stages the same build to asf-staging (release-docs-stage.py's fetch/copy/commit/push), unless --no-stage
Usage:
python3 scripts/push.py "Commit message" [--no-stage] [--dry-run]
[--juneau-root DIR] [--console full|board|none]
Options:
--no-stage Gate, commit and push only: build with --skip-maven (no Maven site) and do not stage
--dry-run Print the steps and run the Prechecks; build, commit and push nothing
--juneau-root Juneau source checkout (default: $JUNEAU_ROOT, then sibling ../master, then ../juneau)
--console Console output when the Juneau checkout has juneau-terminal: full, board (default) or none
--stage-message Commit message for the asf-staging commit (default: "Deploy documentation staging")
Never promotes to production; that stays release-docs.py.
"""
REQUIRED_BRANCH = 'docs'
AI_CHECK = 'check-ai-artifacts.py'
AI_GENERATE = 'generate-ai-artifacts.py'
AI_ROW = 'ai-artifacts'
def load_sibling(filename):
"""Import a hyphenated sibling script (release-docs-stage.py) as a module."""
path = Path(__file__).parent / filename
spec = importlib.util.spec_from_file_location(filename.replace('-', '_').removesuffix('.py'), path)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
return module
def git_out(repo, *args):
"""(exit code, stdout, stderr) of a git command in repo."""
result = subprocess.run(['git', *args], cwd=repo, capture_output=True, text=True, check=False)
return result.returncode, result.stdout, result.stderr
def staged_paths(repo):
"""Paths in the index that differ from HEAD."""
code, out, err = git_out(repo, 'diff', '--cached', '--name-only')
if code != 0:
raise RuntimeError(err.strip() or 'git diff --cached failed')
return [line for line in out.splitlines() if line]
def unreviewed_paths(repo):
"""Tracked files with unstaged changes plus untracked files that are not gitignored."""
paths = []
for args in (('diff', '--name-only'), ('ls-files', '--others', '--exclude-standard')):
code, out, err = git_out(repo, *args)
if code != 0:
raise RuntimeError(err.strip() or f"git {args[0]} failed")
paths += [line for line in out.splitlines() if line]
return sorted(set(paths))
def branch_problem(repo):
"""None on branch docs; otherwise (status, reason, hints)."""
code, out, _ = git_out(repo, 'symbolic-ref', '--short', '-q', 'HEAD')
branch = out.strip() if code == 0 else None
if branch != REQUIRED_BRANCH:
return ('fail', f"on {branch or 'a detached HEAD'}, not {REQUIRED_BRANCH}",
(f"git checkout {REQUIRED_BRANCH}",))
return None
def unreviewed_problem(repo):
"""None when nothing is unstaged or untracked; otherwise (status, reason, hints) listing the paths."""
try:
paths = unreviewed_paths(repo)
except RuntimeError as e:
return ('fail', f"could not check the working tree: {e}", ())
if not paths:
return None
hint = [*paths[:20], *([f"... and {len(paths) - 20} more"] if len(paths) > 20 else []),
"Staging is how this script knows a path was reviewed: stage each one you have read, or set it aside",
"(git stash push -- <paths>, then git stash pop after). Nothing was built, committed or pushed."]
return ('fail', f"{len(paths)} unreviewed path{'s' if len(paths) != 1 else ''} (not staged)", tuple(hint))
def upstream_problem(repo):
"""None when docs has an upstream it is not behind; otherwise (status, reason, hints)."""
warning = terminal_view.branch_warning(repo)
if warning is not None:
return ('fail', warning, ("git pull --rebase", "Nothing was built, committed or pushed."))
code, _, _ = git_out(repo, 'rev-parse', '--abbrev-ref', '--symbolic-full-name', '@{upstream}')
if code != 0:
return ('fail', f"{REQUIRED_BRANCH} has no upstream", (f"git push -u origin {REQUIRED_BRANCH}",))
return None
def ahead_count(repo):
"""Commits on HEAD that the upstream does not have."""
code, out, _ = git_out(repo, 'rev-list', '--count', '@{upstream}..HEAD')
return int(out.strip()) if code == 0 and out.strip().isdigit() else 0
def precheck_list(stage, docs_dir):
"""The Prechecks row's (name, fn) pairs, in order."""
return [("Git identity (apache.org email)", lambda: stage.identity_problem(docs_dir)),
("Docs branch", lambda: branch_problem(docs_dir)),
("Unreviewed changes", lambda: unreviewed_problem(docs_dir)),
("Upstream", lambda: upstream_problem(docs_dir))]
def push_plan(args):
"""The run's top-level rows, in order, as (id, label)."""
plan = [('prechecks', 'Prechecks'), (AI_ROW, 'AI artifacts'), *terminal_view.BUILD_ROWS, ('commit', 'Commit'), ('push', 'Push')]
if not args.no_stage:
plan += [('stage', 'Stage push'), ('url', 'URL')]
return plan
def fail(view, message, *hints):
"""Report a failure and return the failing exit code."""
view.say(f"❌ {message}", 'error')
for hint in hints:
view.say(f" {hint}", 'error')
return 1
def commit_and_push(view, docs_dir, message):
"""Commit the index as found (if anything is staged) and push to the upstream; returns the exit code."""
try:
staged = staged_paths(docs_dir)
except RuntimeError as e:
return fail(view, f"Could not read the index: {e}")
ahead = ahead_count(docs_dir)
if staged:
view.cmd(['git', 'commit', '-q', '-m', message], cwd=docs_dir, id='commit', label='Commit',
title=f"Committing {len(staged)} staged path{'s' if len(staged) != 1 else ''}", tool='generic')
ahead += 1
else:
view.skip('commit', 'Commit', "commit", "nothing staged")
if ahead:
with view.paused(): # a credential or passphrase prompt shows on a cleared screen
view.cmd(['git', 'push'], cwd=docs_dir, id='push', label='Push', title=f"Pushing {REQUIRED_BRANCH}",
tool='generic', tty=True)
else:
view.skip('push', 'Push', "push", "nothing to push")
return 0
def ai_artifact_path(path):
"""True for a path the AI-artifacts generator may write: static/llms.txt and anything under static/ai/."""
return path == 'static/llms.txt' or path.startswith('static/ai/')
def regenerate_ai_artifacts(view, docs_dir, juneau_root):
"""
Regenerate stale AI artifacts and stage exactly what that run wrote; returns (exit code, staged paths).
This is the one thing this script stages, and the test is provenance: only files the generator itself wrote in this
run, after the unreviewed-changes refusal has passed (so every unstaged or untracked path now is that output). It
refuses, staging nothing, when the generator wrote anything outside static/llms.txt and static/ai/, or when the
operator had already staged one of those paths (unstaging on a failed run would drop their review).
"""
check = docs_dir / 'scripts' / AI_CHECK
generate = docs_dir / 'scripts' / AI_GENERATE
if juneau_root is None or not check.exists() or not generate.exists():
view.skip(AI_ROW, 'AI artifacts', "AI artifacts", "no Juneau checkout")
return 0, []
argv = ['--juneau-root', str(juneau_root)]
def run(script):
return subprocess.run([sys.executable, '-B', str(script), *argv], cwd=docs_dir, capture_output=True,
text=True, check=False)
if run(check).returncode == 0:
view.skip(AI_ROW, 'AI artifacts', "AI artifacts", "up to date")
return 0, []
try:
already = [p for p in staged_paths(docs_dir) if ai_artifact_path(p)]
before = set(unreviewed_paths(docs_dir))
except RuntimeError as e:
return fail(view, f"Could not read the git state: {e}"), []
if already:
return fail(view, "The AI artifacts are stale, but you have staged some of them already: " + ", ".join(already),
f"Regenerate them yourself (python3 scripts/{AI_GENERATE}) or unstage them, then run again."), []
with view.stage(AI_ROW, 'AI artifacts', "Regenerating AI artifacts"):
result = run(generate)
if result.returncode != 0:
return fail(view, f"scripts/{AI_GENERATE} failed (exit {result.returncode}). Nothing was committed or pushed.",
*(result.stdout + result.stderr).strip().splitlines()[-10:]), []
try:
written = sorted(set(unreviewed_paths(docs_dir)) - before)
except RuntimeError as e:
return fail(view, f"Could not read the git state: {e}"), []
stray = [p for p in written if not ai_artifact_path(p)]
if stray:
return fail(view, f"scripts/{AI_GENERATE} wrote outside static/llms.txt and static/ai/; nothing was staged:",
*stray), []
if run(check).returncode != 0:
return fail(view, "The AI artifacts are still stale after regenerating them. Nothing was committed or pushed."), []
if written:
code, _, err = git_out(docs_dir, 'add', '--', *written)
if code != 0:
unstage_regenerated(docs_dir, written)
return fail(view, f"git add failed: {err.strip()}"), []
view.say(f"Regenerated {len(written)} AI artifact file{'s' if len(written) != 1 else ''} and staged "
f"{'it' if len(written) == 1 else 'them'} for this commit:")
for path in written:
view.say(f" {path}")
return 0, written
JAVADOC_PREFIX = 'static/javadocs/'
def stage_built_javadocs(view, docs_dir, staged_before):
"""
Stage the javadoc output the build wrote (new, modified or deleted files under static/javadocs/); returns
(exit code, paths newly staged by this run).
Provenance is the same as for the AI artifacts: the unreviewed-changes refusal passed before the build, so every
unstaged or untracked path now is build output. A path the operator had staged before the run is re-added (it is
generated) but not returned, so a failed run never unstages it.
"""
try:
built = [p for p in unreviewed_paths(docs_dir) if p.startswith(JAVADOC_PREFIX)]
except RuntimeError as e:
return fail(view, f"Could not read the git state: {e}"), []
if not built:
return 0, []
# -f: static/.gitignore ignores javadocs/, and git add refuses a pathspec under an ignored directory even when the
# file is tracked. Every path here came from git diff or a non-ignored ls-files, so -f adds no ignored file.
code, _, err = git_out(docs_dir, 'add', '-A', '-f', '--', *built)
if code != 0:
unstage_regenerated(docs_dir, [p for p in built if p not in staged_before])
return fail(view, f"git add failed: {err.strip()}"), []
view.say(f"Staged {len(built)} regenerated javadoc file{'s' if len(built) != 1 else ''} for this commit.")
return 0, [p for p in built if p not in staged_before]
def unstage_regenerated(docs_dir, paths):
"""Unstage exactly the paths regenerate_ai_artifacts staged (a no-op once they are committed); the files stay on disk."""
if paths:
git_out(docs_dir, 'restore', '--staged', '--', *paths)
def push_steps(args, view, stage, docs_dir, juneau_root):
"""
Prechecks, the AI-artifacts regeneration, the build gate, commit, push and (unless --no-stage) the staging deploy;
returns the exit code. A run that ends in failure (or an exception) unstages whatever it regenerated and staged.
"""
regenerated = []
try:
code = _run_steps(args, view, stage, docs_dir, juneau_root, regenerated)
except BaseException:
unstage_regenerated(docs_dir, regenerated)
raise
if code:
unstage_regenerated(docs_dir, regenerated)
return code
def _run_steps(args, view, stage, docs_dir, juneau_root, regenerated):
"""The steps of push_steps; appends to `regenerated` the paths it staged."""
view.banner("Prechecks")
if not view.prechecks(precheck_list(stage, docs_dir)):
return 1
view.banner("AI artifacts")
code, paths = regenerate_ai_artifacts(view, docs_dir, juneau_root)
regenerated.extend(paths)
if code:
return code
try:
staged_before = set(staged_paths(docs_dir))
except RuntimeError as e:
return fail(view, f"Could not read the index: {e}")
view.banner("Building and checking the documentation")
if args.no_stage:
if not (stage.BUILD_SCRIPT).exists():
return fail(view, f"ERROR: {stage.BUILD_SCRIPT} not found")
cmd = [sys.executable, str(stage.BUILD_SCRIPT), '--skip-maven']
if args.juneau_root is not None:
cmd += ['--juneau-root', str(juneau_root)]
if args.console is not None:
cmd += ['--console', args.console]
with view.handoff() as extra:
code = subprocess.run(cmd, cwd=docs_dir, env={**os.environ, **extra}).returncode
if code != 0:
return fail(view, "The documentation build or its checks failed. Nothing was committed or pushed.")
else:
code = stage.build_for_staging(view, docs_dir, juneau_root, juneau_root_given=args.juneau_root is not None,
console=args.console)
if code is not None:
return code
# The build copies fresh javadocs into static/javadocs/: stage them so they ride in this commit. Any other tracked
# file it rewrote (artifact maps, the javadoc index) is build output, not reviewed work: reported, never committed.
code, paths = stage_built_javadocs(view, docs_dir, staged_before)
regenerated.extend(paths)
if code:
return code
try:
built = unreviewed_paths(docs_dir)
except RuntimeError:
built = []
if built:
view.say(f"⚠ The build left {len(built)} changed path{'s' if len(built) != 1 else ''} in the tree "
f"(not committed): {', '.join(built[:5])}{', ...' if len(built) > 5 else ''}", 'warn')
problem = upstream_problem(docs_dir)
if problem is not None:
return fail(view, f"After the build: {problem[1]}", *problem[2])
code = commit_and_push(view, docs_dir, args.message)
if code:
return code
if args.no_stage:
view.say("✅ Pushed. Staging skipped (--no-stage).")
return 0
stage_args = argparse.Namespace(no_push=False, commit_message=args.stage_message)
return stage.publish_build(stage_args, view, docs_dir)
def dry_run(args, view, stage, docs_dir):
"""Print the steps and run the Prechecks; returns the exit code."""
print("Steps that would be executed:")
print(f" 1. Prechecks: identity {stage.REQUIRED_GIT_EMAIL}, branch {REQUIRED_BRANCH}, no unreviewed "
"(unstaged/untracked) changes, not behind upstream")
print(f" 2. python3 scripts/{AI_CHECK}; when stale, python3 scripts/{AI_GENERATE}, then stage only the "
"files that run rewrote (static/llms.txt, static/ai/; refuses if it wrote anything else; unstaged again if "
"the run fails before the commit)")
print(f" 3. python3 scripts/build-docs.py {'--skip-maven' if args.no_stage else '--staging'} "
"(every docs check, then the Docusaurus build)")
print(f" 3b. Stage the javadoc output the build wrote under {JAVADOC_PREFIX} (unstaged again if the run fails "
"before the commit; other build output is left unstaged)")
print(f" 4. Commit whatever is staged: git commit -m \"{args.message}\" (stages nothing else)")
print(f" 5. Push {REQUIRED_BRANCH} to its upstream")
if args.no_stage:
print(" 6. Staging skipped (--no-stage)")
else:
print(" 6. Stage the same build: fetch ../asf-staging, copy build/, commit, force-push asf-staging")
ok = view.prechecks(precheck_list(stage, docs_dir))
print("\nDry run complete. Use without --dry-run to execute.")
return 0 if ok else 1
def resolve_message(message):
"""The commit message, prompted for when omitted; exits 1 when empty."""
if message:
return message
try:
message = input("Commit message: ").strip()
except (EOFError, KeyboardInterrupt):
message = ''
if not message:
print("❌ Aborted: a commit message is required.")
sys.exit(1)
return message
def main():
parser = argparse.ArgumentParser(
description="Gate, commit, push and stage the Juneau documentation",
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog=_HELP_EPILOG
)
parser.add_argument('message', nargs='?', help='Git commit message; prompted for if omitted')
parser.add_argument('--no-stage', action='store_true',
help='Gate, commit and push only: build with --skip-maven and do not stage')
parser.add_argument('--dry-run', action='store_true',
help='Print the steps and run the Prechecks; build, commit and push nothing')
parser.add_argument('--stage-message', default='Deploy documentation staging',
help='Commit message for the asf-staging commit (default: "Deploy documentation staging")')
parser.add_argument('--juneau-root', type=Path, default=None,
help='Juneau source checkout (default: $JUNEAU_ROOT, then sibling ../master, then ../juneau)')
terminal_view.add_console_argument(parser)
args = parser.parse_args()
stage = load_sibling('release-docs-stage.py')
docs_dir = Path(__file__).parent.absolute().parent
juneau_root = find_juneau_root(docs_dir, args.juneau_root, allow_missing=True, validate=False)
if not args.no_stage and (juneau_root is None or not Path(juneau_root).is_dir()):
print("❌ A staged push needs the Juneau checkout for the Maven site and javadocs; "
"pass --juneau-root or use --no-stage.")
sys.exit(1)
view = terminal_view.open_view(juneau_root, args.console, plain=args.dry_run)
if args.dry_run:
args.message = args.message or '<message>'
sys.exit(dry_run(args, view, stage, docs_dir))
args.message = resolve_message(args.message)
if view.live:
header = ["Juneau Docs", REQUIRED_BRANCH]
if args.no_stage:
header.append("--no-stage: not staged")
code = view.session("Push", lambda: push_steps(args, view, stage, docs_dir, juneau_root),
header=header, lines=[f"Commit Message: {args.message}", f"Docs: {docs_dir}"],
plan=push_plan(args), repo=docs_dir, mode='push')
else:
print("=" * 79)
print("Push Documentation")
print("=" * 79)
try:
code = push_steps(args, view, stage, docs_dir, juneau_root)
except subprocess.CalledProcessError as e:
print(f"\n❌ Command failed (exit code: {e.returncode}): {' '.join(map(str, e.cmd))}")
code = 1
except KeyboardInterrupt:
print("\n\n⚠️ Process interrupted by user")
code = 130
except Exception as e:
print(f"\n❌ ERROR: {e}")
traceback.print_exc()
code = 1
stage.play_sound(success=code == 0)
sys.exit(code)
if __name__ == '__main__':
main()