blob: 0f8134642e268d892266b102a869343d967b7f42 [file]
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Python code generation for the ``tvm-ffi-stubgen`` tool.
This module owns the Python codegen orchestration for language-agnostic FFI
metadata (:class:`tvm_ffi.stub.utils.FuncInfo` /
:class:`~tvm_ffi.stub.utils.ObjectInfo`). Rendering helpers live in
``python_generator.utils`` so the per-block generation pipeline here stays focused
on directive handling and source assembly.
"""
from __future__ import annotations
from typing import Callable
from .. import consts as C
from ..file_utils import CodeBlock
from ..utils import FuncInfo, InitConfig, ObjectInfo, Options
from . import consts as PC
from .utils import (
ImportItem,
render_func_signature,
render_object_ffi_init,
render_object_fields,
render_object_init,
render_object_methods,
)
# --- Python scaffolding templates (init mode) -------------------------------
# These emit Python source plus stub-directive markers. The marker comment token
# comes from the supplied `MarkerSyntax`, so the directive structure stays
# language-aware even though the surrounding code is Python-specific.
def _prompt_globals(mod: str, syntax: C.MarkerSyntax) -> str:
return f"""{syntax.begin} global/{mod}
{syntax.end}
"""
def _prompt_class_def(
type_name: str, type_key: str, parent_type_name: str, syntax: C.MarkerSyntax
) -> str:
return f'''@_FFI_REG_OBJ("{type_key}")
class {type_name}({parent_type_name}):
"""FFI binding for `{type_key}`."""
{syntax.begin} object/{type_key}
{syntax.end}\n\n'''
def _prompt_import_object(type_key: str, type_name: str, syntax: C.MarkerSyntax) -> str:
return f"""{syntax.import_object} {type_key};False;{type_name}\n"""
def _prompt_import_section(syntax: C.MarkerSyntax) -> str:
return f"""
{syntax.begin} import-section
{syntax.end}
"""
def _prompt_all_section(syntax: C.MarkerSyntax) -> str:
return f"""
__all__ = [
{syntax.begin} __all__
{syntax.end}
]
"""
def _type_suffix_and_record(
ty_map: dict[str, str],
imports: list[ImportItem],
func_names: set[str] | None = None,
) -> Callable[[str], str]:
def _run(name: str) -> str:
nonlocal ty_map, imports
name = ty_map.get(name, name)
suffix = name.rsplit(".", 1)[-1]
if "." in name:
alias = None
if func_names and suffix in func_names:
alias = f"_{suffix}"
imports.append(ImportItem(name, type_checking_only=True, alias=alias))
if alias:
return alias
return suffix
return _run
def _make_input_ty_map(ty_map: dict[str, str]) -> dict[str, str]:
"""Derive input-side defaults without overriding explicit ty-map entries."""
input_ty_map = ty_map.copy()
for key, input_default in PC.TY_MAP_INPUT_DEFAULTS.items():
output_default = PC.TY_MAP_DEFAULTS.get(key)
if ty_map.get(key, output_default) == output_default:
input_ty_map[key] = input_default
return input_ty_map
def generate_python_global_funcs(
code: CodeBlock,
global_funcs: list[FuncInfo],
ty_map: dict[str, str],
imports: list[ImportItem],
opt: Options,
) -> None:
"""Generate function signatures for global functions.
It processes: global/${prefix}@${import_from="tvm_ffi")
"""
assert len(code.lines) >= 2
if not global_funcs:
return
assert isinstance(code.param, tuple)
prefix, import_from = code.param
if not import_from:
import_from = "tvm_ffi"
imports.extend(
[
ImportItem(
f"{import_from}.init_ffi_api",
type_checking_only=False,
alias="_FFI_INIT_FUNC",
),
ImportItem(
"typing.TYPE_CHECKING",
type_checking_only=False,
),
]
)
func_names = {f.schema.name.rsplit(".", 1)[-1] for f in global_funcs}
fn_ty_map = _type_suffix_and_record(ty_map, imports, func_names=func_names)
input_fn_ty_map = _type_suffix_and_record(
_make_input_ty_map(ty_map), imports, func_names=func_names
)
results: list[str] = [
"# fmt: off",
f'_FFI_INIT_FUNC("{prefix}", __name__)',
"if TYPE_CHECKING:",
*[
render_func_signature(func, fn_ty_map, opt.indent, input_ty_map=input_fn_ty_map)
for func in global_funcs
],
"# fmt: on",
]
indent = " " * code.indent
code.lines = [
code.lines[0],
*[indent + line for line in results],
code.lines[-1],
]
def generate_python_object(
code: CodeBlock,
ty_map: dict[str, str],
imports: list[ImportItem],
opt: Options,
obj_info: ObjectInfo,
) -> None:
"""Generate a class definition for an object type.
It processes: object/${type_key}
"""
assert len(code.lines) >= 2
info = obj_info
method_names = {m.schema.name.rsplit(".", 1)[-1] for m in info.methods}
fn_ty_map = _type_suffix_and_record(ty_map, imports, func_names=method_names)
input_fn_ty_map = _type_suffix_and_record(
_make_input_ty_map(ty_map), imports, func_names=method_names
)
init_lines = render_object_init(info, fn_ty_map, opt.indent, input_ty_map=input_fn_ty_map)
ffi_init_lines = render_object_ffi_init(
info, fn_ty_map, opt.indent, input_ty_map=input_fn_ty_map
)
type_checking_lines = [
*init_lines,
*ffi_init_lines,
*render_object_methods(info, fn_ty_map, opt.indent, input_ty_map=input_fn_ty_map),
]
if type_checking_lines:
imports.append(
ImportItem(
"typing.TYPE_CHECKING",
type_checking_only=False,
)
)
results = [
"# fmt: off",
*render_object_fields(info, fn_ty_map, 0),
"if TYPE_CHECKING:",
*type_checking_lines,
"# fmt: on",
]
else:
results = [
"# fmt: off",
*render_object_fields(info, fn_ty_map, 0),
"# fmt: on",
]
indent = " " * code.indent
code.lines = [
code.lines[0],
*[indent + line for line in results],
code.lines[-1],
]
def generate_python_import_section(
code: CodeBlock,
imports: list[ImportItem],
opt: Options,
) -> None:
"""Generate import statements for the types used in the stub.
It processes: import-section
"""
imports_concrete: dict[str, list[ImportItem]] = {}
imports_ty_check: dict[str, list[ImportItem]] = {}
for item in imports:
if item.type_checking_only:
imports_ty_check.setdefault(item.mod, []).append(item)
else:
imports_concrete.setdefault(item.mod, []).append(item)
if imports_ty_check:
imports_concrete.setdefault("typing", []).append(
ImportItem("typing.TYPE_CHECKING", type_checking_only=True)
)
def _make_line(mod: str, items: list[ImportItem], indent: int) -> str:
items.sort(key=lambda item: item.name)
names = ", ".join(sorted(set(item.name_with_alias for item in items)))
indent_str = " " * indent
if mod:
return f"{indent_str}from {mod} import {names}"
else:
return f"{indent_str}import {names}"
results: list[str] = []
if imports_concrete:
results.extend(
_make_line(mod, imports_concrete[mod], indent=0) for mod in sorted(imports_concrete)
)
if imports_ty_check:
results.append("if TYPE_CHECKING:")
results.extend(
_make_line(mod, imports_ty_check[mod], opt.indent) for mod in sorted(imports_ty_check)
)
if results:
code.lines = [
code.lines[0],
"# fmt: off",
"# isort: off",
"from __future__ import annotations",
*results,
"# isort: on",
"# fmt: on",
code.lines[-1],
]
def generate_python_all(code: CodeBlock, names: set[str], opt: Options) -> None:
"""Generate an `__all__` variable for the given names."""
assert len(code.lines) >= 2
if not names:
return
indent = " " * code.indent
names = {f.rsplit(".", 1)[-1] for f in names}
def _sort_key(name: str) -> tuple[int, str]:
if name.isupper():
return (0, name)
if name and name[0].isupper() and not "_" in name:
return (1, name)
return (2, name)
code.lines = [
code.lines[0],
*[f'{indent}"{name}",' for name in sorted(names, key=_sort_key)],
code.lines[-1],
]
def generate_python_export(code: CodeBlock) -> None:
"""Generate an `__all__` variable for the given names."""
assert len(code.lines) >= 2
mod = code.param
code.lines = [
code.lines[0],
"# fmt: off",
"# isort: off",
f"from .{mod} import * # noqa: F403",
f"from .{mod} import __all__ as {mod}__all__",
'if "__all__" not in globals():',
" __all__ = []",
f"__all__.extend({mod}__all__)",
"# isort: on",
"# fmt: on",
code.lines[-1],
]
def generate_python_ffi_api(
code_blocks: list[CodeBlock],
ty_map: dict[str, str],
module_name: str,
object_infos: list[ObjectInfo],
init_cfg: InitConfig,
is_root: bool,
syntax: C.MarkerSyntax,
) -> str:
"""Generate the initial FFI API stub code for a given module."""
# TODO(@junrus): New code is appended to the end of the file.
# We should consider a more sophisticated approach.
append = ""
# Part 0. Imports
if not code_blocks:
append += f"""\"\"\"FFI API bindings for {module_name}.\"\"\"\n"""
if not any(code.kind == "import-section" for code in code_blocks):
append += _prompt_import_section(syntax)
# Part 1. Library loading
if is_root:
append += _prompt_import_object("tvm_ffi.libinfo.load_lib_module", "_FFI_LOAD_LIB", syntax)
append += f"""LIB = _FFI_LOAD_LIB("{init_cfg.pkg}", "{init_cfg.shared_target}")\n"""
# Part 2. Global functions
if not any(code.kind == "global" for code in code_blocks):
append += _prompt_globals(module_name, syntax)
# Part 3. Object types
if object_infos:
append += _prompt_import_object("tvm_ffi.register_object", "_FFI_REG_OBJ", syntax)
defined_type_keys = {
ty_map.get(info.type_key, info.type_key) for info in object_infos if info.type_key
}
for info in object_infos:
type_key = info.type_key
parent_type_key = info.parent_type_key
if type_key is None:
continue
# Canonicalize type key names
type_key = ty_map.get(type_key, type_key)
type_name = type_key.rsplit(".", 1)[-1]
parent_type_key = (
ty_map.get(parent_type_key, parent_type_key) if parent_type_key else parent_type_key
)
parent_type_name = parent_type_key.rsplit(".", 1)[-1] if parent_type_key else "Object"
# Import parent type keys if they are not defined in the current module
if parent_type_key and parent_type_key not in defined_type_keys:
parent_type_name = "_" + parent_type_key.replace(".", "_")
append += _prompt_import_object(parent_type_key, parent_type_name, syntax)
# Generate class definition
append += _prompt_class_def(
type_name,
type_key,
parent_type_name,
syntax,
)
# Part 4. __all__
if not any(code.kind == "__all__" for code in code_blocks):
append += _prompt_all_section(syntax)
return append
def generate_python_init(
code_blocks: list[CodeBlock],
module_name: str,
submodule: str,
syntax: C.MarkerSyntax,
) -> str:
"""Generate the `__init__.py` file for the `tvm_ffi` package."""
code = f"""
{syntax.begin} export/{submodule}
{syntax.end}
"""
if not code_blocks:
return f"""\"\"\"Package {module_name}.\"\"\"\n""" + code
if not any(code.kind == "export" for code in code_blocks):
return code
return ""