blob: 774d161bccdf06764db330a37d316ea64fcbdffd [file]
#!/usr/bin/env python3
# ***************************************************************************************************************************
# * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file *
# * distributed with this work for additional information regarding copyright ownership. The ASF licenses this file *
# * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance *
# * with the License. You may obtain a copy of the License at *
# * *
# * http://www.apache.org/licenses/LICENSE-2.0 *
# * *
# * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an *
# * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the *
# * specific language governing permissions and limitations under the License. *
# ***************************************************************************************************************************
"""
Generate deterministic AI-consumable knowledge artifacts for Apache Juneau under static/ai.
Full docs: generate-ai-artifacts.md
"""
from __future__ import annotations
import argparse
import hashlib
import json
import re
import subprocess
import sys
from collections import defaultdict
from dataclasses import dataclass
from pathlib import Path
from typing import Any
import project_index
TOPIC_GLOB = "pages/topics/*.md"
RELEASE_NOTES_GLOB = "pages/release-notes/*.md"
ABOUT_GLOB = "src/pages/about/*.mdx"
SIDEBARS_FILE = "sidebars.ts"
SITE_URL = "https://juneau.apache.org"
LLMS_SUMMARY_MAX = 200
SCHEMA_VERSION = "1.0.0"
REQUIRED_FIELDS = [
"id",
"title",
"summary",
"description",
"module",
"source_type",
"source_path",
"version",
"tags",
"related_ids",
"updated_at",
]
# Manifest keys that record where the artifacts were generated from rather than what they contain.
# --check ignores them, so a new commit in either repo with no content change is not "stale".
CHECK_IGNORED_MANIFEST_KEYS = ("source_commit",)
@dataclass
class ExtractConfig:
docs_root: Path
juneau_root: Path
output_dir: Path
def parse_args() -> argparse.Namespace:
script_dir = Path(__file__).resolve().parent
docs_root = script_dir.parent
default_juneau_root = docs_root.parent / "juneau"
default_output_dir = docs_root / "static" / "ai"
parser = argparse.ArgumentParser(description="Generate AI artifacts for Juneau docs.")
parser.add_argument("--docs-root", type=Path, default=docs_root)
parser.add_argument("--juneau-root", type=Path, default=default_juneau_root)
parser.add_argument("--output-dir", type=Path, default=default_output_dir)
parser.add_argument("--check", action="store_true", help="Verify generated output is up to date without writing.")
return parser.parse_args()
def read_text(path: Path) -> str:
return path.read_text(encoding="utf-8")
def get_git_commit(repo_root: Path) -> str:
try:
result = subprocess.run(
["git", "rev-parse", "HEAD"],
cwd=repo_root,
capture_output=True,
text=True,
check=True,
)
return result.stdout.strip()
except Exception:
return "unknown"
def get_current_release(juneau_root: Path) -> str:
script = juneau_root / "scripts" / "current-release.py"
if not script.exists():
return "unknown"
try:
result = subprocess.run(
[sys.executable, str(script)],
cwd=juneau_root,
capture_output=True,
text=True,
check=True,
)
return result.stdout.strip() or "unknown"
except Exception:
return "unknown"
def parse_frontmatter(content: str) -> dict[str, str]:
m = re.match(r"^---\n(.*?)\n---\n?", content, re.DOTALL)
if not m:
return {}
frontmatter = {}
for line in m.group(1).splitlines():
if ":" not in line:
continue
key, value = line.split(":", 1)
frontmatter[key.strip()] = value.strip().strip("\"'")
return frontmatter
def remove_frontmatter(content: str) -> str:
return re.sub(r"^---\n.*?\n---\n?", "", content, count=1, flags=re.DOTALL)
def first_heading(content: str) -> str:
for line in content.splitlines():
if line.startswith("# "):
return line[2:].strip()
return ""
def first_paragraph(content: str) -> str:
cleaned = remove_frontmatter(content)
lines = [ln.rstrip() for ln in cleaned.splitlines()]
bucket = []
for line in lines:
stripped = line.strip()
if not stripped:
if bucket:
break
continue
if stripped.startswith("#") or stripped.startswith("```") or stripped.startswith(">"):
continue
bucket.append(stripped)
if len(" ".join(bucket)) > 240:
break
return " ".join(bucket).strip()
def short_digest(parts: list[str]) -> str:
payload = "|".join(parts)
return hashlib.sha1(payload.encode("utf-8")).hexdigest()[:12]
def build_doc_record(path: Path, docs_root: Path, source_type: str, default_version: str) -> dict[str, Any]:
content = read_text(path)
fm = parse_frontmatter(content)
rel_path = str(path.relative_to(docs_root))
title = fm.get("title") or first_heading(content) or path.stem
summary = first_paragraph(content) or title
description = remove_frontmatter(content).strip()
if len(description) > 2000:
description = description[:1997] + "..."
module = "general"
if source_type == "topic":
m = re.match(r"^\d+[a-z]?\.([^.]+)", path.name)
if m:
module = m.group(1)
if source_type == "release-note":
module = "release-notes"
version = default_version
if source_type == "release-note":
version = path.stem
return {
"id": f"{source_type}:{short_digest([rel_path, title])}",
"title": title,
"summary": summary,
"description": description,
"module": module,
"source_type": source_type,
"source_path": rel_path,
"version": version,
"tags": [source_type, module],
"related_ids": [],
"updated_at": "source-controlled",
}
def about_body(content: str) -> str:
"""An about page's Markdown without front matter, import lines and the <ProjectLayout> wrapper tags."""
body = remove_frontmatter(content)
body = re.sub(r"^import .*\n", "", body, flags=re.MULTILINE)
body = re.sub(r"^\s*</?ProjectLayout\b[^>]*>\s*\n", "", body, flags=re.MULTILINE)
return body.strip()
def build_about_records(config: ExtractConfig, current_release: str) -> list[dict[str, Any]]:
"""One record per /about project page: `about-<slug>`. `index.mdx` is the redirect stub and gets no record."""
pages = sorted(p for p in config.docs_root.glob(ABOUT_GLOB) if p.stem != "index")
if not pages:
return []
_, projects = project_index.load(config.docs_root)
by_slug = {p["slug"]: p for p in projects}
records = []
for path in pages:
content = read_text(path)
fm = parse_frontmatter(content)
body = about_body(content)
project = by_slug.get(path.stem)
if project is not None:
record_id, title, summary = f"about-{project['slug']}", project["name"], project["tagline"]
module = project["modules"][0]
# The module line comes before the body so the 2000-character cut can never drop it.
description = "\n\n".join([title, summary, "Modules: " + ", ".join(project["modules"]), body])
else:
record_id, title, summary = "about", fm.get("title") or path.stem, fm.get("description") or ""
module = "general"
description = body
if len(description) > 2000:
description = description[:1997] + "..."
records.append({
"id": record_id,
"title": title,
"summary": summary,
"description": description,
"module": module,
"source_type": "about",
"source_path": str(path.relative_to(config.docs_root)),
"version": current_release,
"tags": ["about", module],
"related_ids": [],
"updated_at": "source-controlled",
})
return records
def extract_api_metadata_records(juneau_root: Path, default_version: str) -> list[dict[str, Any]]:
patterns = [
r"\bsummary\s*\(\)\s*default\s*\"\"",
r"\bsu\s*\(\)\s*default\s*\"\"",
]
include_roots = [
juneau_root / "juneau-core",
juneau_root / "juneau-rest",
]
records = []
for root in include_roots:
if not root.exists():
continue
for path in sorted(root.rglob("*.java")):
text = read_text(path)
if not any(re.search(pattern, text) for pattern in patterns):
continue
rel_path = str(path.relative_to(juneau_root))
symbol = path.stem
records.append(
{
"id": f"api-metadata:{short_digest([rel_path, symbol])}",
"title": f"{symbol} summary/su metadata",
"summary": f"{symbol} exposes summary/su metadata fields for concise AI-oriented descriptions.",
"description": (
"Detected summary()/su() annotation attributes used by Juneau schema and HTTP metadata "
"annotations. These fields provide short descriptions intended for compact AI/LLM consumption."
),
"module": "api-metadata",
"source_type": "api-metadata",
"source_path": rel_path,
"version": default_version,
"tags": ["api-metadata", "summary", "su"],
"related_ids": [],
"updated_at": "source-controlled",
}
)
return records
def parse_sidebar_tree(sidebars_text: str) -> list[dict[str, Any]]:
"""Tokenize the mainSidebar array of sidebars.ts into a tree of {type,label,id,link,children} dicts."""
start = re.search(r"\bmainSidebar\s*:", sidebars_text)
if start is None:
raise ValueError("Could not find 'mainSidebar' in sidebars.ts")
sidebars_text = sidebars_text[start.end():]
token_re = re.compile(
r"(?P<link>\blink\s*:\s*\{)|(?P<open>\{)|(?P<close>\})"
r"|\b(?P<key>type|label|id)\s*:\s*'(?P<val>(?:[^'\\]|\\.)*)'"
)
root: dict[str, Any] = {"children": []}
stack: list[dict[str, Any]] = [root]
kinds: list[str] = ["root"]
for m in token_re.finditer(sidebars_text):
if m.group("link") or m.group("open"):
node: dict[str, Any] = {"children": []}
stack.append(node)
kinds.append("link" if m.group("link") else "obj")
elif m.group("close"):
if len(stack) == 1:
continue
node = stack.pop()
kind = kinds.pop()
if kind == "link":
stack[-1]["link"] = node
else:
stack[-1]["children"].append(node)
else:
stack[-1][m.group("key")] = m.group("val").replace("\\'", "'")
# The mainSidebar array's single object wrapper is the root's only meaningful content.
return root["children"]
def find_node(nodes: list[dict[str, Any]], label: str) -> dict[str, Any] | None:
for n in nodes:
if n.get("label") == label and n.get("type") == "category":
return n
found = find_node(n.get("children", []), label)
if found:
return found
return None
def doc_ids_in_order(node: dict[str, Any]) -> list[str]:
"""Depth-first doc ids for a category: its own link page first, then its items."""
ids: list[str] = []
link = node.get("link")
if link and link.get("id"):
ids.append(link["id"])
for child in node.get("children", []):
if child.get("type") == "category":
ids.extend(doc_ids_in_order(child))
elif child.get("id"):
ids.append(child["id"])
return ids
def topic_url(record: dict[str, Any], docs_root: Path) -> str:
fm = parse_frontmatter(read_text(docs_root / record["source_path"]))
slug = fm.get("slug") or Path(record["source_path"]).stem
return f"{SITE_URL}/docs/topics/{slug}"
def one_line(text: str, limit: int = LLMS_SUMMARY_MAX) -> str:
text = re.sub(r"<[^>]+>", "", text)
text = re.sub(r"\[([^\]]*)\]\([^)]*\)", r"\1", text)
text = text.replace("**", "")
text = re.sub(r"\s+", " ", text).strip()
if len(text) <= limit:
return text
cut = text[: limit - 3].rsplit(" ", 1)[0].rstrip(",;:.")
return cut + "..."
def render_llms_txt(config: ExtractConfig, records: list[dict[str, Any]], current_release: str) -> str:
docs_root = config.docs_root
by_id: dict[str, dict[str, Any]] = {}
for rec in records:
if rec["source_type"] != "topic":
continue
fm = parse_frontmatter(read_text(docs_root / rec["source_path"]))
stem = Path(rec["source_path"]).stem
by_id["topics/" + (fm.get("id") or stem)] = rec
tree = parse_sidebar_tree(read_text(docs_root / SIDEBARS_FILE))
top = {n.get("label"): n for n in tree if n.get("type") == "category"}
for name in ("Getting Started", "Guides", "Documentation"):
if name not in top:
raise ValueError(f"Could not find '{name}' category in sidebars.ts")
sections = [top["Getting Started"], top["Guides"], *top["Documentation"]["children"]]
lines = [
"# Apache Juneau",
"",
"> Apache Juneau is a Java toolkit for marshalling POJOs to and from many formats (JSON, XML, HTML, "
"UON, URL-encoding, MessagePack, RDF, CSV, YAML and more), building REST servers and clients, "
"and writing configurable microservices, built around a common bean-context and annotation model. "
"This index links to the documentation topics, grouped as in the Getting Started, Guides and Documentation sections of the site sidebar.",
"",
]
seen: set[str] = set()
for section in sections:
if section.get("type") != "category":
continue
entries = []
for doc_id in doc_ids_in_order(section):
rec = by_id.get(doc_id)
if rec is None or doc_id in seen:
continue
seen.add(doc_id)
summary = one_line(rec["summary"])
entries.append(f"- [{rec['title']}]({topic_url(rec, docs_root)}): {summary}")
if entries:
lines.extend([f"## {section['label']}", "", *entries, ""])
about = {r["id"]: r for r in records if r["source_type"] == "about"}
if about:
lines.extend(["## About", ""])
for project in project_index.load(docs_root)[1]:
rec = about.get(f"about-{project['slug']}")
if rec is not None:
lines.append(f"- [{rec['title']}]({SITE_URL}/about/{project['slug']}/): {one_line(rec['summary'])}")
lines.append("")
migration = next((r for r in records if r["source_path"].endswith("27.V10MigrationGuide.md")), None)
lines.extend(["## Resources", ""])
lines.append(f"- [Release notes {current_release}]({SITE_URL}/docs/release-notes/{current_release}): "
"What changed in the current release.")
if migration:
lines.append(f"- [{migration['title']}]({topic_url(migration, docs_root)}): "
"Upgrading from Juneau 9.x to 10.0.")
lines.append(f"- [Javadocs {current_release}]({SITE_URL}/javadocs/{current_release}/index.html): "
"API reference for the current release.")
lines.append(f"- [Knowledge base (JSONL)]({SITE_URL}/ai/juneau-knowledge.jsonl): "
"Machine-readable records of all topics, release notes, About project pages and API metadata, for RAG pipelines.")
lines.append(f"- [Knowledge base manifest]({SITE_URL}/ai/manifest.json): "
"Version, record count and source commits for the knowledge base.")
lines.append("")
return "\n".join(lines)
def build_records(config: ExtractConfig, current_release: str) -> list[dict[str, Any]]:
records = []
for path in sorted(config.docs_root.glob(TOPIC_GLOB)):
records.append(build_doc_record(path, config.docs_root, "topic", current_release))
for path in sorted(config.docs_root.glob(RELEASE_NOTES_GLOB)):
records.append(build_doc_record(path, config.docs_root, "release-note", current_release))
records.extend(build_about_records(config, current_release))
records.extend(extract_api_metadata_records(config.juneau_root, current_release))
records.sort(key=lambda r: (r["source_type"], r["source_path"], r["id"]))
return records
def validate_records(records: list[dict[str, Any]]) -> None:
for record in records:
missing = [field for field in REQUIRED_FIELDS if field not in record]
if missing:
raise ValueError(f"Record missing required fields {missing}: {record.get('id')}")
if not isinstance(record["tags"], list):
raise ValueError(f"Record tags must be a list: {record['id']}")
if not isinstance(record["related_ids"], list):
raise ValueError(f"Record related_ids must be a list: {record['id']}")
def build_taxonomy(records: list[dict[str, Any]]) -> dict[str, Any]:
by_module: dict[str, int] = defaultdict(int)
by_type: dict[str, int] = defaultdict(int)
for record in records:
by_module[record["module"]] += 1
by_type[record["source_type"]] += 1
return {
"schema_version": SCHEMA_VERSION,
"modules": dict(sorted(by_module.items())),
"source_types": dict(sorted(by_type.items())),
}
def compute_generated_at(records: list[dict[str, Any]]) -> str:
digest = short_digest([record["id"] for record in records])
return f"content-hash:{digest}"
def build_manifest(
docs_commit: str,
juneau_commit: str,
current_release: str,
records: list[dict[str, Any]],
) -> dict[str, Any]:
return {
"schema_version": SCHEMA_VERSION,
"version": current_release,
"generated_at": compute_generated_at(records),
"source_commit": {
"juneau_docs": docs_commit,
"juneau": juneau_commit,
},
"record_count": len(records),
}
def render_jsonl(records: list[dict[str, Any]]) -> str:
return "".join(json.dumps(record, sort_keys=True, ensure_ascii=True) + "\n" for record in records)
def render_json(value: dict[str, Any]) -> str:
return json.dumps(value, indent=2, sort_keys=True, ensure_ascii=True) + "\n"
def is_stale(path: Path, actual: str | None, expected: str) -> bool:
"""Byte-compare an artifact, except manifest.json keys in CHECK_IGNORED_MANIFEST_KEYS."""
if actual is None:
return True
if path.name != "manifest.json":
return actual != expected
try:
actual_manifest = json.loads(actual)
except json.JSONDecodeError:
return True
if not isinstance(actual_manifest, dict):
return True
expected_manifest = json.loads(expected)
for key in CHECK_IGNORED_MANIFEST_KEYS:
if key in actual_manifest:
expected_manifest[key] = actual_manifest[key]
return actual != render_json(expected_manifest)
def write_if_changed(path: Path, content: str) -> bool:
existing = path.read_text(encoding="utf-8") if path.exists() else None
if existing == content:
return False
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
return True
def main() -> int:
args = parse_args()
config = ExtractConfig(
docs_root=args.docs_root.resolve(),
juneau_root=args.juneau_root.resolve(),
output_dir=args.output_dir.resolve(),
)
current_release = get_current_release(config.juneau_root)
docs_commit = get_git_commit(config.docs_root)
juneau_commit = get_git_commit(config.juneau_root)
records = build_records(config, current_release)
validate_records(records)
taxonomy = build_taxonomy(records)
manifest = build_manifest(docs_commit, juneau_commit, current_release, records)
outputs = {
config.output_dir / "juneau-knowledge.jsonl": render_jsonl(records),
config.output_dir / "taxonomy.json": render_json(taxonomy),
config.output_dir / "manifest.json": render_json(manifest),
config.output_dir.parent / "llms.txt": render_llms_txt(config, records, current_release),
}
if args.check:
stale = []
for path, expected in outputs.items():
actual = path.read_text(encoding="utf-8") if path.exists() else None
if is_stale(path, actual, expected):
stale.append(path)
if stale:
print("ERROR: AI artifacts are stale. Regenerate with:")
print(" python3 scripts/generate-ai-artifacts.py")
for path in stale:
print(f" - {path}")
return 1
print("AI artifacts are up to date.")
return 0
changed = 0
for path, content in outputs.items():
if write_if_changed(path, content):
changed += 1
print(f"Updated {path}")
else:
print(f"No change {path}")
print(f"Done. {changed} file(s) updated.")
return 0
if __name__ == "__main__":
raise SystemExit(main())