| """Creates reST files corresponding to Python modules for code documentation. |
| |
| Parses a directory tree looking for Python modules and packages and creates |
| ReST files appropriately to create code documentation with Sphinx. It also |
| creates a modules index (named modules.<suffix>). |
| |
| This is derived from the "sphinx-autopackage" script, which is: |
| Copyright 2008 Société des arts technologiques (SAT), |
| https://sat.qc.ca/ |
| """ |
| |
| from __future__ import annotations |
| |
| import argparse |
| import fnmatch |
| import glob |
| import locale |
| import os |
| import re |
| import sys |
| from copy import copy |
| from importlib.machinery import EXTENSION_SUFFIXES |
| from os import path |
| from typing import TYPE_CHECKING, Any |
| |
| import sphinx.locale |
| from sphinx import __display_version__, package_dir |
| from sphinx.cmd.quickstart import EXTENSIONS |
| from sphinx.locale import __ |
| from sphinx.util import logging |
| from sphinx.util.osutil import FileAvoidWrite, ensuredir |
| from sphinx.util.template import ReSTRenderer |
| |
| if TYPE_CHECKING: |
| from collections.abc import Generator, Sequence |
| |
| logger = logging.getLogger(__name__) |
| |
| # automodule options |
| if 'SPHINX_APIDOC_OPTIONS' in os.environ: |
| OPTIONS = os.environ['SPHINX_APIDOC_OPTIONS'].split(',') |
| else: |
| OPTIONS = [ |
| 'members', |
| 'undoc-members', |
| # 'inherited-members', # disabled because there's a bug in sphinx |
| 'show-inheritance', |
| ] |
| |
| PY_SUFFIXES = ('.py', '.pyx') + tuple(EXTENSION_SUFFIXES) |
| |
| template_dir = path.join(package_dir, 'templates', 'apidoc') |
| |
| |
| def is_initpy(filename: str) -> bool: |
| """Check *filename* is __init__ file or not.""" |
| basename = path.basename(filename) |
| return any( |
| basename == '__init__' + suffix |
| for suffix in sorted(PY_SUFFIXES, key=len, reverse=True) |
| ) |
| |
| |
| def module_join(*modnames: str | None) -> str: |
| """Join module names with dots.""" |
| return '.'.join(filter(None, modnames)) |
| |
| |
| def is_packagedir(dirname: str | None = None, files: list[str] | None = None) -> bool: |
| """Check given *files* contains __init__ file.""" |
| if files is None and dirname is None: |
| return False |
| |
| if files is None: |
| files = os.listdir(dirname) |
| return any(f for f in files if is_initpy(f)) |
| |
| |
| def write_file(name: str, text: str, opts: Any) -> None: |
| """Write the output file for module/package <name>.""" |
| quiet = getattr(opts, 'quiet', None) |
| |
| fname = path.join(opts.destdir, f'{name}.{opts.suffix}') |
| if opts.dryrun: |
| if not quiet: |
| logger.info(__('Would create file %s.'), fname) |
| return |
| if not opts.force and path.isfile(fname): |
| if not quiet: |
| logger.info(__('File %s already exists, skipping.'), fname) |
| else: |
| if not quiet: |
| logger.info(__('Creating file %s.'), fname) |
| with FileAvoidWrite(fname) as f: |
| f.write(text) |
| |
| |
| def create_module_file(package: str | None, basename: str, opts: Any, |
| user_template_dir: str | None = None) -> None: |
| """Build the text of the file and write the file.""" |
| options = copy(OPTIONS) |
| if opts.includeprivate and 'private-members' not in options: |
| options.append('private-members') |
| |
| qualname = module_join(package, basename) |
| context = { |
| 'show_headings': not opts.noheadings, |
| 'basename': basename, |
| 'qualname': qualname, |
| 'automodule_options': options, |
| } |
| if user_template_dir is not None: |
| template_path = [user_template_dir, template_dir] |
| else: |
| template_path = [template_dir] |
| text = ReSTRenderer(template_path).render('module.rst_t', context) |
| write_file(qualname, text, opts) |
| |
| |
| def create_package_file(root: str, master_package: str | None, subroot: str, |
| py_files: list[str], |
| opts: Any, subs: list[str], is_namespace: bool, |
| excludes: Sequence[re.Pattern[str]] = (), |
| user_template_dir: str | None = None, |
| ) -> None: |
| """Build the text of the file and write the file.""" |
| # build a list of sub packages (directories containing an __init__ file) |
| subpackages = [module_join(master_package, subroot, pkgname) |
| for pkgname in subs |
| if not is_skipped_package(path.join(root, pkgname), opts, excludes)] |
| # build a list of sub modules |
| submodules = [sub.split('.')[0] for sub in py_files |
| if not is_skipped_module(path.join(root, sub), opts, excludes) and |
| not is_initpy(sub)] |
| submodules = sorted(set(submodules)) |
| submodules = [module_join(master_package, subroot, modname) |
| for modname in submodules] |
| options = copy(OPTIONS) |
| if opts.includeprivate and 'private-members' not in options: |
| options.append('private-members') |
| |
| pkgname = module_join(master_package, subroot) |
| context = { |
| 'pkgname': pkgname, |
| 'subpackages': subpackages, |
| 'submodules': submodules, |
| 'is_namespace': is_namespace, |
| 'modulefirst': opts.modulefirst, |
| 'separatemodules': opts.separatemodules, |
| 'automodule_options': options, |
| 'show_headings': not opts.noheadings, |
| 'maxdepth': opts.maxdepth, |
| } |
| if user_template_dir is not None: |
| template_path = [user_template_dir, template_dir] |
| else: |
| template_path = [template_dir] |
| text = ReSTRenderer(template_path).render('package.rst_t', context) |
| write_file(pkgname, text, opts) |
| |
| if submodules and opts.separatemodules: |
| for submodule in submodules: |
| create_module_file(None, submodule, opts, user_template_dir) |
| |
| |
| def create_modules_toc_file(modules: list[str], opts: Any, name: str = 'modules', |
| user_template_dir: str | None = None) -> None: |
| """Create the module's index.""" |
| modules.sort() |
| prev_module = '' |
| for module in modules[:]: |
| # look if the module is a subpackage and, if yes, ignore it |
| if module.startswith(prev_module + '.'): |
| modules.remove(module) |
| else: |
| prev_module = module |
| |
| context = { |
| 'header': opts.header, |
| 'maxdepth': opts.maxdepth, |
| 'docnames': modules, |
| } |
| if user_template_dir is not None: |
| template_path = [user_template_dir, template_dir] |
| else: |
| template_path = [template_dir] |
| text = ReSTRenderer(template_path).render('toc.rst_t', context) |
| write_file(name, text, opts) |
| |
| |
| def is_skipped_package(dirname: str, opts: Any, |
| excludes: Sequence[re.Pattern[str]] = ()) -> bool: |
| """Check if we want to skip this module.""" |
| if not path.isdir(dirname): |
| return False |
| |
| files = glob.glob(path.join(dirname, '*.py')) |
| regular_package = any(f for f in files if is_initpy(f)) |
| if not regular_package and not opts.implicit_namespaces: |
| # *dirname* is not both a regular package and an implicit namespace package |
| return True |
| |
| # Check there is some showable module inside package |
| return all(is_excluded(path.join(dirname, f), excludes) for f in files) |
| |
| |
| def is_skipped_module(filename: str, opts: Any, _excludes: Sequence[re.Pattern[str]]) -> bool: |
| """Check if we want to skip this module.""" |
| if not path.exists(filename): |
| # skip if the file doesn't exist |
| return True |
| if path.basename(filename).startswith('_') and not opts.includeprivate: |
| # skip if the module has a "private" name |
| return True |
| return False |
| |
| |
| def walk(rootpath: str, excludes: Sequence[re.Pattern[str]], opts: Any, |
| ) -> Generator[tuple[str, list[str], list[str]], None, None]: |
| """Walk through the directory and list files and subdirectories up.""" |
| followlinks = getattr(opts, 'followlinks', False) |
| includeprivate = getattr(opts, 'includeprivate', False) |
| |
| for root, subs, files in os.walk(rootpath, followlinks=followlinks): |
| # document only Python module files (that aren't excluded) |
| files = sorted(f for f in files |
| if f.endswith(PY_SUFFIXES) and |
| not is_excluded(path.join(root, f), excludes)) |
| |
| # remove hidden ('.') and private ('_') directories, as well as |
| # excluded dirs |
| if includeprivate: |
| exclude_prefixes: tuple[str, ...] = ('.',) |
| else: |
| exclude_prefixes = ('.', '_') |
| |
| subs[:] = sorted(sub for sub in subs if not sub.startswith(exclude_prefixes) and |
| not is_excluded(path.join(root, sub), excludes)) |
| |
| yield root, subs, files |
| |
| |
| def has_child_module(rootpath: str, excludes: Sequence[re.Pattern[str]], opts: Any) -> bool: |
| """Check the given directory contains child module/s (at least one).""" |
| return any( |
| files |
| for _root, _subs, files in walk(rootpath, excludes, opts) |
| ) |
| |
| |
| def recurse_tree(rootpath: str, excludes: Sequence[re.Pattern[str]], opts: Any, |
| user_template_dir: str | None = None) -> list[str]: |
| """ |
| Look for every file in the directory tree and create the corresponding |
| ReST files. |
| """ |
| implicit_namespaces = getattr(opts, 'implicit_namespaces', False) |
| |
| # check if the base directory is a package and get its name |
| if is_packagedir(rootpath) or implicit_namespaces: |
| root_package = rootpath.split(path.sep)[-1] |
| else: |
| # otherwise, the base is a directory with packages |
| root_package = None |
| |
| toplevels = [] |
| for root, subs, files in walk(rootpath, excludes, opts): |
| is_pkg = is_packagedir(None, files) |
| is_namespace = not is_pkg and implicit_namespaces |
| if is_pkg: |
| for f in files[:]: |
| if is_initpy(f): |
| files.remove(f) |
| files.insert(0, f) |
| elif root != rootpath: |
| # only accept non-package at toplevel unless using implicit namespaces |
| if not implicit_namespaces: |
| del subs[:] |
| continue |
| |
| if is_pkg or is_namespace: |
| # we are in a package with something to document |
| if subs or len(files) > 1 or not is_skipped_package(root, opts): |
| subpackage = root[len(rootpath):].lstrip(path.sep).\ |
| replace(path.sep, '.') |
| # if this is not a namespace or |
| # a namespace and there is something there to document |
| if not is_namespace or has_child_module(root, excludes, opts): |
| create_package_file(root, root_package, subpackage, |
| files, opts, subs, is_namespace, excludes, |
| user_template_dir) |
| toplevels.append(module_join(root_package, subpackage)) |
| else: |
| # if we are at the root level, we don't require it to be a package |
| assert root == rootpath |
| assert root_package is None |
| for py_file in files: |
| if not is_skipped_module(path.join(rootpath, py_file), opts, excludes): |
| module = py_file.split('.')[0] |
| create_module_file(root_package, module, opts, user_template_dir) |
| toplevels.append(module) |
| |
| return toplevels |
| |
| |
| def is_excluded(root: str, excludes: Sequence[re.Pattern[str]]) -> bool: |
| """Check if the directory is in the exclude list. |
| |
| Note: by having trailing slashes, we avoid common prefix issues, like |
| e.g. an exclude "foo" also accidentally excluding "foobar". |
| """ |
| return any(exclude.match(root) for exclude in excludes) |
| |
| |
| def get_parser() -> argparse.ArgumentParser: |
| parser = argparse.ArgumentParser( |
| usage='%(prog)s [OPTIONS] -o <OUTPUT_PATH> <MODULE_PATH> ' |
| '[EXCLUDE_PATTERN, ...]', |
| epilog=__('For more information, visit <https://www.sphinx-doc.org/>.'), |
| description=__(""" |
| Look recursively in <MODULE_PATH> for Python modules and packages and create |
| one reST file with automodule directives per package in the <OUTPUT_PATH>. |
| |
| The <EXCLUDE_PATTERN>s can be file and/or directory patterns that will be |
| excluded from generation. |
| |
| Note: By default this script will not overwrite already created files.""")) |
| |
| parser.add_argument('--version', action='version', dest='show_version', |
| version='%%(prog)s %s' % __display_version__) |
| |
| parser.add_argument('module_path', |
| help=__('path to module to document')) |
| parser.add_argument('exclude_pattern', nargs='*', |
| help=__('fnmatch-style file and/or directory patterns ' |
| 'to exclude from generation')) |
| |
| parser.add_argument('-o', '--output-dir', action='store', dest='destdir', |
| required=True, |
| help=__('directory to place all output')) |
| parser.add_argument('-q', action='store_true', dest='quiet', |
| help=__('no output on stdout, just warnings on stderr')) |
| parser.add_argument('-d', '--maxdepth', action='store', dest='maxdepth', |
| type=int, default=4, |
| help=__('maximum depth of submodules to show in the TOC ' |
| '(default: 4)')) |
| parser.add_argument('-f', '--force', action='store_true', dest='force', |
| help=__('overwrite existing files')) |
| parser.add_argument('-l', '--follow-links', action='store_true', |
| dest='followlinks', default=False, |
| help=__('follow symbolic links. Powerful when combined ' |
| 'with collective.recipe.omelette.')) |
| parser.add_argument('-n', '--dry-run', action='store_true', dest='dryrun', |
| help=__('run the script without creating files')) |
| parser.add_argument('-e', '--separate', action='store_true', |
| dest='separatemodules', |
| help=__('put documentation for each module on its own page')) |
| parser.add_argument('-P', '--private', action='store_true', |
| dest='includeprivate', |
| help=__('include "_private" modules')) |
| parser.add_argument('--tocfile', action='store', dest='tocfile', default='modules', |
| help=__("filename of table of contents (default: modules)")) |
| parser.add_argument('-T', '--no-toc', action='store_false', dest='tocfile', |
| help=__("don't create a table of contents file")) |
| parser.add_argument('-E', '--no-headings', action='store_true', |
| dest='noheadings', |
| help=__("don't create headings for the module/package " |
| "packages (e.g. when the docstrings already " |
| "contain them)")) |
| parser.add_argument('-M', '--module-first', action='store_true', |
| dest='modulefirst', |
| help=__('put module documentation before submodule ' |
| 'documentation')) |
| parser.add_argument('--implicit-namespaces', action='store_true', |
| dest='implicit_namespaces', |
| help=__('interpret module paths according to PEP-0420 ' |
| 'implicit namespaces specification')) |
| parser.add_argument('-s', '--suffix', action='store', dest='suffix', |
| default='rst', |
| help=__('file suffix (default: rst)')) |
| parser.add_argument('-F', '--full', action='store_true', dest='full', |
| help=__('generate a full project with sphinx-quickstart')) |
| parser.add_argument('-a', '--append-syspath', action='store_true', |
| dest='append_syspath', |
| help=__('append module_path to sys.path, used when --full is given')) |
| parser.add_argument('-H', '--doc-project', action='store', dest='header', |
| help=__('project name (default: root module name)')) |
| parser.add_argument('-A', '--doc-author', action='store', dest='author', |
| help=__('project author(s), used when --full is given')) |
| parser.add_argument('-V', '--doc-version', action='store', dest='version', |
| help=__('project version, used when --full is given')) |
| parser.add_argument('-R', '--doc-release', action='store', dest='release', |
| help=__('project release, used when --full is given, ' |
| 'defaults to --doc-version')) |
| |
| group = parser.add_argument_group(__('extension options')) |
| group.add_argument('--extensions', metavar='EXTENSIONS', dest='extensions', |
| action='append', help=__('enable arbitrary extensions')) |
| for ext in EXTENSIONS: |
| group.add_argument('--ext-%s' % ext, action='append_const', |
| const='sphinx.ext.%s' % ext, dest='extensions', |
| help=__('enable %s extension') % ext) |
| |
| group = parser.add_argument_group(__('Project templating')) |
| group.add_argument('-t', '--templatedir', metavar='TEMPLATEDIR', |
| dest='templatedir', |
| help=__('template directory for template files')) |
| |
| return parser |
| |
| |
| def main(argv: Sequence[str] = (), /) -> int: |
| """Parse and check the command line arguments.""" |
| locale.setlocale(locale.LC_ALL, '') |
| sphinx.locale.init_console() |
| |
| parser = get_parser() |
| args = parser.parse_args(argv or sys.argv[1:]) |
| |
| rootpath = path.abspath(args.module_path) |
| |
| # normalize opts |
| |
| if args.header is None: |
| args.header = rootpath.split(path.sep)[-1] |
| if args.suffix.startswith('.'): |
| args.suffix = args.suffix[1:] |
| if not path.isdir(rootpath): |
| logger.error(__('%s is not a directory.'), rootpath) |
| raise SystemExit(1) |
| if not args.dryrun: |
| ensuredir(args.destdir) |
| excludes = tuple( |
| re.compile(fnmatch.translate(path.abspath(exclude))) |
| for exclude in dict.fromkeys(args.exclude_pattern) |
| ) |
| modules = recurse_tree(rootpath, excludes, args, args.templatedir) |
| |
| if args.full: |
| from sphinx.cmd import quickstart as qs |
| modules.sort() |
| prev_module = '' |
| text = '' |
| for module in modules: |
| if module.startswith(prev_module + '.'): |
| continue |
| prev_module = module |
| text += ' %s\n' % module |
| d = { |
| 'path': args.destdir, |
| 'sep': False, |
| 'dot': '_', |
| 'project': args.header, |
| 'author': args.author or 'Author', |
| 'version': args.version or '', |
| 'release': args.release or args.version or '', |
| 'suffix': '.' + args.suffix, |
| 'master': 'index', |
| 'epub': True, |
| 'extensions': ['sphinx.ext.autodoc', 'sphinx.ext.viewcode', |
| 'sphinx.ext.todo'], |
| 'makefile': True, |
| 'batchfile': True, |
| 'make_mode': True, |
| 'mastertocmaxdepth': args.maxdepth, |
| 'mastertoctree': text, |
| 'language': 'en', |
| 'module_path': rootpath, |
| 'append_syspath': args.append_syspath, |
| } |
| if args.extensions: |
| d['extensions'].extend(args.extensions) |
| if args.quiet: |
| d['quiet'] = True |
| |
| for ext in d['extensions'][:]: |
| if ',' in ext: |
| d['extensions'].remove(ext) |
| d['extensions'].extend(ext.split(',')) |
| |
| if not args.dryrun: |
| qs.generate(d, silent=True, overwrite=args.force, |
| templatedir=args.templatedir) |
| elif args.tocfile: |
| create_modules_toc_file(modules, args, args.tocfile, args.templatedir) |
| |
| return 0 |
| |
| |
| # So program can be started with "python -m sphinx.apidoc ..." |
| if __name__ == "__main__": |
| raise SystemExit(main(sys.argv[1:])) |
