| #!/usr/bin/env python3 | |
| # *************************************************************************************************************************** | |
| # * Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file * | |
| # * distributed with this work for additional information regarding copyright ownership. The ASF licenses this file * | |
| # * to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance * | |
| # * with the License. You may obtain a copy of the License at * | |
| # * * | |
| # * http://www.apache.org/licenses/LICENSE-2.0 * | |
| # * * | |
| # * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an * | |
| # * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the * | |
| # * specific language governing permissions and limitations under the License. * | |
| # *************************************************************************************************************************** | |
| """ | |
| Check that src/data/projects.ts, the /about MDX pages and their links agree (checks C1 to C13). | |
| Full docs: check-project-pages.md | |
| """ | |
| import argparse | |
| import json | |
| import re | |
| import sys | |
| from pathlib import Path | |
| import project_index | |
| from project_old_names import find_old_names | |
| ABOUT_DIR = 'src/pages/about' | |
| SLUG_RE = re.compile(r'[a-z0-9]+(-[a-z0-9]+)*') | |
| LAYOUT_TAG_RE = re.compile(r'<ProjectLayout\b([^>]*)>', re.S) | |
| REDIRECT_HOME_RE = re.compile(r'<Redirect\b[^>]*\bto\s*=\s*(?:"/"|\'/\'|\{\s*"/"\s*\}|\{\s*\'/\'\s*\})') | |
| TOPIC_LINK_RE = re.compile(r'/docs/topics/([A-Za-z0-9_.-]+)') | |
| FRONT_MATTER_RE = re.compile(r'\A---\n(.*?)\n---', re.S) | |
| FM_KEY_RE = re.compile(r'^(title|description|hide_table_of_contents):\s*(.*?)\s*$', re.M) | |
| def _unquote(value): | |
| if len(value) >= 2 and value[0] == value[-1] and value[0] in '\'"': | |
| inner = value[1:-1] | |
| return inner.replace("''", "'") if value[0] == "'" else inner | |
| return value | |
| def _front_matter(text): | |
| m = FRONT_MATTER_RE.match(text) | |
| if not m: | |
| return {} | |
| return {k: _unquote(v) for k, v in FM_KEY_RE.findall(m.group(1))} | |
| def _topic_slugs(docs_root): | |
| slugs = set() | |
| for page in Path(docs_root, 'pages', 'topics').glob('*.md'): | |
| m = FRONT_MATTER_RE.match(page.read_text(encoding='utf-8')) | |
| slug = re.search(r'^slug:\s*(\S+)', m.group(1), re.M) if m else None | |
| slugs.add(slug.group(1) if slug else page.stem) | |
| return slugs | |
| def _layout_slug_ok(attrs, slug): | |
| q = re.escape(slug) | |
| return re.search(rf'\bslug\s*=\s*(?:"{q}"|\'{q}\'|\{{\s*"{q}"\s*\}}|\{{\s*\'{q}\'\s*\}})', attrs) is not None | |
| def _check_layout(rel, text, slug, errors): | |
| tags = LAYOUT_TAG_RE.findall(text) | |
| if len(tags) != 1: | |
| errors.append(f'C3: {rel}: expected exactly one <ProjectLayout>, found {len(tags)}') | |
| elif not _layout_slug_ok(tags[0], slug): | |
| errors.append(f'C3: {rel}: <ProjectLayout> slug must be "{slug}"') | |
| def _check_redirect_stub(rel, text, errors): | |
| if LAYOUT_TAG_RE.search(text): | |
| errors.append(f'C3: {rel}: the redirect stub must not contain a <ProjectLayout>') | |
| if not REDIRECT_HOME_RE.search(text): | |
| errors.append(f'C3: {rel}: the redirect stub must contain <Redirect to="/" />') | |
| if 'http-equiv="refresh"' not in text or 'name="robots"' not in text or 'noindex' not in text: | |
| errors.append(f'C3: {rel}: the redirect stub needs <meta http-equiv="refresh"> and ' | |
| f'<meta name="robots" content="noindex"> for visitors without JavaScript') | |
| def _line_of(text, offset): | |
| return text.count('\n', 0, offset) + 1 | |
| def check(docs_root): | |
| """Every violation as 'C<n>: <where>: <what>'; empty list when the project pages are consistent.""" | |
| docs_root = Path(docs_root) | |
| try: | |
| groups, projects = project_index.load(docs_root) | |
| module_docs = project_index.load_module_docs(docs_root) | |
| except project_index.ProjectIndexError as e: | |
| return [f'C0: {project_index.PROJECTS_FILE}: {e}'] | |
| errors = [] | |
| about = docs_root / ABOUT_DIR | |
| pages = {p.stem: p for p in sorted(about.glob('*.mdx'))} | |
| overview = pages.pop('index', None) | |
| slugs = [p.get('slug', '') for p in projects] | |
| # C4, C1, C2: slugs are unique kebab-case and match the page files one to one. | |
| seen = set() | |
| for slug in slugs: | |
| if slug in seen: | |
| errors.append(f'C4: {project_index.PROJECTS_FILE}: duplicate slug "{slug}"') | |
| seen.add(slug) | |
| if not SLUG_RE.fullmatch(slug): | |
| errors.append(f'C4: {project_index.PROJECTS_FILE}: slug "{slug}" is not kebab-case') | |
| for slug in sorted(set(slugs) - set(pages)): | |
| errors.append(f'C1: {ABOUT_DIR}/{slug}.mdx: project "{slug}" has no page') | |
| for stem in sorted(set(pages) - set(slugs)): | |
| errors.append(f'C2: {ABOUT_DIR}/{stem}.mdx: page has no project in {project_index.PROJECTS_FILE}') | |
| # C3: each project page wraps its content in one <ProjectLayout> with the matching slug; index.mdx is the | |
| # redirect stub to the home page and has no layout. | |
| for stem, page in pages.items(): | |
| _check_layout(f'{ABOUT_DIR}/{stem}.mdx', page.read_text(encoding='utf-8'), stem, errors) | |
| if overview is None: | |
| errors.append(f'C3: {ABOUT_DIR}/index.mdx: the redirect stub is missing') | |
| else: | |
| _check_redirect_stub(f'{ABOUT_DIR}/index.mdx', overview.read_text(encoding='utf-8'), errors) | |
| # C5, C10: every project's group exists, every group has a project, and the order is group by group. | |
| group_ids = [g.get('id', '') for g in groups] | |
| used = {p.get('group') for p in projects} | |
| for p in projects: | |
| if p.get('group') not in group_ids: | |
| errors.append(f'C5: {project_index.PROJECTS_FILE}: project "{p.get("slug")}" has unknown group "{p.get("group")}"') | |
| for gid in group_ids: | |
| if gid not in used: | |
| errors.append(f'C5: {project_index.PROJECTS_FILE}: group "{gid}" has no project') | |
| order = [group_ids.index(p['group']) for p in projects if p.get('group') in group_ids] | |
| if order != sorted(order): | |
| errors.append(f'C10: {project_index.PROJECTS_FILE}: projects must be contiguous per group, in GROUPS order') | |
| # C6: modules are real, documentable artifacts (a key in artifact-paths.json that is not an aggregator pom). | |
| try: | |
| keys = set(json.loads(Path(docs_root, 'artifact-paths.json').read_text(encoding='utf-8'))) | |
| except (OSError, ValueError) as e: | |
| keys = set() | |
| errors.append(f'C6: artifact-paths.json: cannot read: {e}') | |
| for p in projects: | |
| for mod in p.get('modules', []): | |
| if mod in project_index.AGGREGATOR_POMS: | |
| errors.append(f'C6: {project_index.PROJECTS_FILE}: project "{p.get("slug")}" lists aggregator pom "{mod}"') | |
| elif mod not in keys: | |
| errors.append(f'C6: {project_index.PROJECTS_FILE}: project "{p.get("slug")}" lists "{mod}", ' | |
| f'which is not a key in artifact-paths.json') | |
| # C7: every Read-more topic and every gettingStarted topic is a real topic slug. | |
| topics = _topic_slugs(docs_root) | |
| for p in projects: | |
| for d in p.get('docs', []): | |
| if d not in topics: | |
| errors.append(f'C7: {project_index.PROJECTS_FILE}: project "{p.get("slug")}" lists topic "{d}", ' | |
| f'which is no pages/topics slug') | |
| gs = p.get('gettingStarted') | |
| if gs is not None and gs not in topics: | |
| errors.append(f'C7: {project_index.PROJECTS_FILE}: project "{p.get("slug")}" lists gettingStarted topic ' | |
| f'"{gs}", which is no pages/topics slug') | |
| # C13: every MODULE_DOCS key is a module some project lists, and every value is a real topic slug. | |
| listed = {m for p in projects for m in p.get('modules', [])} | |
| for mod, slug in module_docs.items(): | |
| if mod not in listed: | |
| errors.append(f'C13: {project_index.PROJECTS_FILE}: MODULE_DOCS key "{mod}" is no project\'s module') | |
| if slug not in topics: | |
| errors.append(f'C13: {project_index.PROJECTS_FILE}: MODULE_DOCS maps "{mod}" to "{slug}", ' | |
| f'which is no pages/topics slug') | |
| # C8: names start with "Juneau " and taglines are not empty. | |
| for p in projects: | |
| if not p.get('name', '').startswith('Juneau '): | |
| errors.append(f'C8: {project_index.PROJECTS_FILE}: name "{p.get("name")}" must start with "Juneau "') | |
| if not p.get('tagline', '').strip(): | |
| errors.append(f'C8: {project_index.PROJECTS_FILE}: project "{p.get("slug")}" has an empty tagline') | |
| # C11: page front matter matches the index (the overview needs only a title and description). | |
| by_slug = {p.get('slug'): p for p in projects} | |
| for stem, page in list(pages.items()) + ([('index', overview)] if overview else []): | |
| rel = f'{ABOUT_DIR}/{stem}.mdx' | |
| fm = _front_matter(page.read_text(encoding='utf-8')) | |
| project = by_slug.get(stem) | |
| if project is not None: | |
| if fm.get('title') != project.get('name'): | |
| errors.append(f'C11: {rel}: title "{fm.get("title")}" must equal the project name "{project.get("name")}"') | |
| if fm.get('description') != project.get('tagline'): | |
| errors.append(f'C11: {rel}: description must equal the project tagline') | |
| elif stem == 'index': | |
| if not fm.get('title') or not fm.get('description'): | |
| errors.append(f'C11: {rel}: needs a title and a description') | |
| if stem in by_slug or stem == 'index': | |
| if fm.get('hide_table_of_contents') != 'true': | |
| errors.append(f'C11: {rel}: needs hide_table_of_contents: true') | |
| # C12: every /docs/topics/ link in a page body is a real topic slug. | |
| for stem, page in list(pages.items()) + ([('index', overview)] if overview else []): | |
| text = page.read_text(encoding='utf-8') | |
| for m in TOPIC_LINK_RE.finditer(text): | |
| if m.group(1).rstrip('.') not in topics: | |
| errors.append(f'C12: {ABOUT_DIR}/{stem}.mdx:{_line_of(text, m.start())}: ' | |
| f'link to unknown topic "{m.group(1)}"') | |
| # C9: no pre-rename name in the pages or the index. | |
| sources = [(f'{ABOUT_DIR}/{stem}.mdx', page) for stem, page in pages.items()] | |
| if overview: | |
| sources.append((f'{ABOUT_DIR}/index.mdx', overview)) | |
| sources.append((project_index.PROJECTS_FILE, docs_root / project_index.PROJECTS_FILE)) | |
| for rel, page in sources: | |
| for label, lineno, hit in find_old_names(page.read_text(encoding='utf-8')): | |
| errors.append(f'C9: {rel}:{lineno}: pre-rename {label} name "{hit}"') | |
| return errors | |
| def main(argv=None): | |
| parser = argparse.ArgumentParser(description='Check the Juneau project pages against src/data/projects.ts.') | |
| parser.add_argument('--docs-dir', default=str(Path(__file__).resolve().parent.parent), | |
| help='the docs checkout (default: the parent of scripts/)') | |
| args = parser.parse_args(argv) | |
| errors = check(args.docs_dir) | |
| for line in errors: | |
| print(line) | |
| if errors: | |
| return 1 | |
| _, projects = project_index.load(args.docs_dir) | |
| print(f'Project pages OK ({len(projects)} projects)') | |
| return 0 | |
| if __name__ == '__main__': | |
| sys.exit(main()) |