docs/source/conf.py

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# https://www.sphinx-doc.org/en/master/usage/configuration.html

# -- Path setup --------------------------------------------------------------

# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))

from pathlib import Path

# docs/, whether this file is read from docs/source/ (a direct sphinx-build) or
# from the docs/temp/ copy that docs/build.sh generates.
DOCS_DIR = Path(__file__).resolve().parent.parent

# -- Project information -----------------------------------------------------

project = 'Apache DataFusion Comet'
copyright = '2023-2024, Apache Software Foundation'
author = 'Apache Software Foundation'


# -- General configuration ---------------------------------------------------

# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = [
    'sphinx.ext.autodoc',
    'sphinx.ext.autosummary',
    'sphinx.ext.doctest',
    'sphinx.ext.ifconfig',
    'sphinx.ext.mathjax',
    'sphinx.ext.viewcode',
    'sphinx.ext.napoleon',
    'myst_parser',
    'sphinx_reredirects',
    # Renders the ```mermaid fences in the contributor guide. See mermaid_output_format
    # below: the diagrams are drawn at build time, not in the reader's browser.
    'sphinxcontrib.mermaid',
]

source_suffix = {
    '.rst': 'restructuredtext',
    '.md': 'markdown',
}

# Route ```mermaid fences to the mermaid directive rather than treating them as a literal
# code block, so the same source renders as a diagram here and on github.com, which
# understands that fence natively.
myst_fence_as_directive = ['mermaid']

# Draw the diagrams at build time with mermaid-cli (mmdc) instead of shipping mermaid.js to
# the browser. The default 'raw' format emits `import mermaid from "https://cdn.jsdelivr.net/..."`,
# and the ASF serves *.apache.org with a Content-Security-Policy whose script-src allows only
# 'self' plus a few apache.org hosts, so that import is blocked and no diagram ever renders on
# the published site. Pre-rendered SVG is served from 'self' and needs no script at all.
# apache/arrow sets this for the same reason.
#
# This makes mmdc a build dependency: `npm install -g @mermaid-js/mermaid-cli`. Without it the
# build still succeeds but logs a warning and drops the diagrams. See docs/README.md.
mermaid_output_format = 'svg'

# -b transparent: render on a transparent background so one SVG suits both the light and dark
# site themes; mmdc otherwise bakes in a white background.
#
# -p puppeteer-config.json: mmdc draws each diagram by driving headless Chrome through puppeteer.
# Chrome's setuid sandbox needs unprivileged user namespaces, which Ubuntu restricts by AppArmor
# policy from 23.10 onwards, so on an ubuntu-24.04 runner Chrome can fail to launch at all. The
# config file turns that sandbox off; the CI container is already the isolation boundary.
#
# Any such failure is invisible without help: mmdc exits non-zero, sphinxcontrib-mermaid turns
# that into a warning and drops the diagram, and the build publishes a page with a hole in it.
# That is issue #6062, and dev/ci/check-mermaid.py is what makes it loud. It reads the arguments
# below rather than repeating them, so the check renders exactly as the build does.
mermaid_params = ['-b', 'transparent', '-p', str(DOCS_DIR / 'puppeteer-config.json')]

# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']

# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = [
    # Per-Spark-version compatibility pages are generated by docs/build.sh from these
    # templates; the templates themselves are not part of the rendered docs.
    'user-guide/latest/compatibility/expressions/_category_template',
]

# Show members for classes in .. autosummary
autodoc_default_options = {
    "members": None,
    "undoc-members": None,
    "show-inheritance": None,
    "inherited-members": None,
}

autosummary_generate = True

# -- Options for HTML output -------------------------------------------------

# The theme to use for HTML and HTML Help pages.  See the documentation for
# a list of builtin themes.
#
html_theme = 'pydata_sphinx_theme'

html_logo = "_static/images/DataFusionComet-Logo-Light.png"

html_theme_options = {
    "logo": {
        "image_light": "_static/images/DataFusionComet-Logo-Light.png",
        "image_dark": "_static/images/DataFusionComet-Logo-Dark.png",
    },

    "use_edit_page_button": False,
    "secondary_sidebar_items": ["page-toc"],
    "collapse_navigation": True,
    "navbar_start": ["navbar-logo"],
    "navbar_center": ["navbar-nav"],
    "navbar_end": ["navbar-icon-links", "theme-switcher"],
    "icon_links": [
        {
            "name": "GitHub",
            "url": "https://github.com/apache/datafusion-comet",
            "icon": "fa-brands fa-github",
        },
    ],

}

html_context = {
    "default_mode": "light",
    "github_user": "apache",
    "github_repo": "datafusion-comet",
    "github_version": "main",
    "doc_path": "docs/source",
}

# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']

html_css_files = [
    "theme_overrides.css"
]

html_js_files = [
    ("https://buttons.github.io/buttons.js", {'async': 'true', 'defer': 'true'}),
    "comet-ux.js",
]

html_sidebars = {
    "**": ["search-field.html", "docs-sidebar.html"],
    # Landing page is a full-width marketing page — no left rail.
    "index": [],
}

# tell myst_parser to auto-generate anchor links for headers h1, h2, h3, h4
myst_heading_anchors = 4

# enable nice rendering of checkboxes for the task lists
myst_enable_extensions = ["colon_fence", "deflist", "tasklist"]

redirects = {
    "overview.html": "index.html",
    "gluten_comparison.html": "about/gluten_comparison.html",
    "user-guide/overview.html": "../index.html",
    "user-guide/gluten_comparison.html": "../about/gluten_comparison.html",
    "user-guide/compatibility.html": "latest/compatibility/index.html",
    "user-guide/configs.html": "latest/configs.html",
    "user-guide/datasources.html": "latest/datasources.html",
    "user-guide/datatypes.html": "latest/datatypes.html",
    "user-guide/expressions.html": "latest/expressions.html",
    "user-guide/iceberg.html": "latest/iceberg.html",
    "user-guide/installation.html": "latest/installation.html",
    "user-guide/kubernetes.html": "latest/kubernetes.html",
    "user-guide/metrics.html": "latest/metrics.html",
    "user-guide/operators.html": "latest/operators.html",
    "user-guide/source.html": "latest/source.html",
    "user-guide/tuning.html": "latest/tuning.html",
    # The compatibility guide became a directory of pages in #4055.
    "user-guide/latest/compatibility.html": "compatibility/index.html",
}


# -- $COMET_VERSION substitution -------------------------------------------
# build.sh runs `replace_in_files` after copying source/ → temp/ to swap the
# $COMET_VERSION placeholder with the real version from pom.xml. When Sphinx
# is invoked directly against source/ (local dev, IDE preview, etc.), that
# substitution never runs and the literal "$COMET_VERSION" leaks into page
# titles, breadcrumbs, and the toctree caption. The hook below mirrors the
# build.sh substitution at Sphinx read-time so local builds match prod.

def _read_comet_version():
    import os
    import xml.etree.ElementTree as ET
    pom_path = os.path.normpath(os.path.join(os.path.dirname(__file__), "..", "..", "pom.xml"))
    try:
        tree = ET.parse(pom_path)
        root = tree.getroot()
        ns = {"m": "http://maven.apache.org/POM/4.0.0"}
        ver = root.find("m:version", ns)
        if ver is not None and ver.text:
            return ver.text.strip()
    except Exception:
        pass
    return "latest"


_comet_version = _read_comet_version()


def _substitute_version(app, docname, source):
    if "$COMET_VERSION" in source[0]:
        source[0] = source[0].replace("$COMET_VERSION", _comet_version)


# -- User Guide sidebar: nest per-version pages under captions --------------
# Every User Guide page groups its pages under captions (Getting Started,
# What Comet Supports, ...) via nested `.. toctree::` blocks in that version's
# own index page — except the tree rendered from user-guide/index itself
# (the version accordion: current version, Development Snapshot, Older
# Versions), where expanding a version used to show a flat list instead of
# the same captioned groups. That's because Sphinx's toctree resolution only
# keeps `:caption:` labels for the page acting as the root of the resolved
# fragment; once a version's toctree is merged as a *child* of the index page
# (two hops away), its captions are dropped. To nest them back in, we resolve
# each version's own toctree separately (as if it were the root, via
# pydata-sphinx-theme's internal toctree helpers) and splice the captioned
# fragment into that version's accordion in the outer, uncaptioned tree.
# Older Versions is left as-is; it isn't touched here.
#
# We render this same nested tree, always anchored at user-guide/index, for
# EVERY page under user-guide/ (not just the index page). Sub-pages used to
# fall back to a different, shallower anchor (the version's own index),
# which dropped the version accordion entirely and replaced the sidebar with
# a completely different tree on every navigation — jarring, and the reason
# an earlier commit on this branch flattened everything back to one
# consistent depth. Keeping one fixed anchor everywhere gets the same
# stability without giving up captions: only the "current" highlighting and
# which accordion is auto-expanded change between pages.
#
# This relies on two undocumented, module-level helpers from
# pydata_sphinx_theme.toctree (get_nonroot_toctree, add_collapse_checkboxes).
# They're pinned via requirements.txt (pydata-sphinx-theme<0.17.0), so a
# theme upgrade won't silently change this behavior.

def _build_user_guide_sidebar_html(app, pagename, context):
    from bs4 import BeautifulSoup
    from pydata_sphinx_theme.toctree import (
        _get_ancestor_pagename,
        add_collapse_checkboxes,
        get_nonroot_toctree,
    )
    from sphinx.environment.adapters.toctree import TocTree

    outer = context["generate_toctree_html"](
        "sidebar", startdepth=1, maxdepth=4, collapse=False,
        includehidden=True, titles_only=True,
    )
    # generate_toctree_html is memoized by the theme and returns a shared,
    # mutable BeautifulSoup object; re-parse into an independent tree so the
    # splicing below can't compound across repeated calls for this page.
    soup = BeautifulSoup(str(outer), "html.parser")

    # Resolve the same depth-1 ancestor generate_toctree_html used above
    # (user-guide/index, regardless of which user-guide/* page is actually
    # being rendered) so we read its toctree's own doc list rather than
    # pagename's — pagename is the page being viewed, not the page that
    # owns the version-accordion toctree.
    toctree = TocTree(app.env)
    ancestorname, _ = _get_ancestor_pagename(app, pagename, startdepth=1)

    # That toctree lists, in order: the current stable version, the
    # development snapshot, then Older Versions. Nest captions into the
    # first two; Older Versions keeps its existing flat list.
    version_docnames = app.env.toctree_includes.get(ancestorname, [])
    top_level_items = soup.select("ul.bd-sidenav > li.toctree-l1")

    for li, docname in zip(top_level_items, version_docnames[:2]):
        details = li.find("details")
        inner_ul = details.find("ul") if details else None
        if inner_ul is None:
            continue
        captioned = get_nonroot_toctree(
            app, pagename, docname, toctree,
            collapse=False, maxdepth=4, includehidden=True, titles_only=True,
        )
        if captioned is None:
            continue
        frag_soup = BeautifulSoup(
            app.builder.render_partial(captioned)["fragment"], "html.parser"
        )
        # The fragment was resolved as its own root, so its items are labeled
        # toctree-l1/-l2/...; shift by one to match the depth they actually
        # sit at once nested inside this version's own l1 entry.
        for tag in frag_soup.find_all(class_=True):
            tag["class"] = [
                f"toctree-l{int(cl[len('toctree-l'):]) + 1}"
                if cl.startswith("toctree-l") and cl[len("toctree-l"):].isdigit()
                else cl
                for cl in tag["class"]
            ]
        add_collapse_checkboxes(frag_soup)
        for child in list(frag_soup.children):
            inner_ul.insert_before(child)
        inner_ul.decompose()

    return str(soup)


def _add_user_guide_sidebar_context(app, pagename, templatename, context, doctree):
    if pagename.startswith("user-guide/"):
        context["user_guide_sidebar_html"] = (
            lambda: _build_user_guide_sidebar_html(app, pagename, context)
        )


def setup(app):
    app.connect("source-read", _substitute_version)
    app.connect("html-page-context", _add_user_guide_sidebar_context)
    return {"version": _comet_version, "parallel_read_safe": True, "parallel_write_safe": True}