| ################################################################################ | |
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| ################################################################################ | |
| """User-defined table functions for the DataFrame API.""" | |
| import collections.abc | |
| import functools | |
| import inspect | |
| import types | |
| from dataclasses import dataclass | |
| from typing import ( | |
| Annotated, Any, Callable, Iterator, List, Optional, Tuple, Type, Union, cast, get_args, | |
| get_origin, overload, | |
| ) | |
| from py4j.java_gateway import get_java_class | |
| from pyflink.common import Row | |
| from pyflink.dataframe.datatype import DataType | |
| from pyflink.dataframe.udf import ( | |
| _DataTypeLike, | |
| _UDFDeclarationContext, | |
| _UNRESOLVED_TYPE_HINT, | |
| _convert_to_dtype, | |
| _create_declaration_context, | |
| _create_result_normalizer, | |
| _get_callable_inspection_target, | |
| _get_callable_return_type_hint, | |
| _is_typed_dict, | |
| _preserves_method_binding, | |
| _resolve_callable_annotation, | |
| _resolve_udf, | |
| _validate_determinism_agreement, | |
| _validate_zero_argument_class, | |
| ) | |
| from pyflink.java_gateway import get_gateway | |
| from pyflink.table.expression import Expression | |
| from pyflink.table.expressions import col, with_columns | |
| from pyflink.table.types import RowType, _to_java_data_type | |
| from pyflink.table.udf import ( | |
| TableFunction, | |
| UserDefinedFunction, | |
| UserDefinedTableFunctionWrapper, | |
| udtf as table_udtf, | |
| ) | |
| from pyflink.util.api_stability_decorators import PublicEvolving | |
| __all__ = ["udtf"] | |
| _UDTFInput = Union[Callable[..., Any], TableFunction, Type] | |
| @dataclass(frozen=True) | |
| class _DataFrameUDTFCall: | |
| expression: Expression | |
| return_dtype: DataType | |
| has_named_fields: bool | |
| output_aliases: Optional[Tuple[str, ...]] = None | |
| def alias(self, name: str, *extra_names: str) -> "_DataFrameUDTFCall": | |
| names = (name,) + extra_names | |
| table_type = self.return_dtype._to_table_data_type() | |
| arity = len(table_type.fields) if isinstance(table_type, RowType) else 1 | |
| if any(not isinstance(value, str) for value in names): | |
| raise TypeError("UDTF output aliases must be strings.") | |
| if len(names) != arity or len(set(names)) != len(names) or not all(names): | |
| raise ValueError( | |
| "UDTF output aliases must be non-empty, unique, and match output arity.") | |
| return _DataFrameUDTFCall( | |
| self.expression.alias(*names), self.return_dtype, self.has_named_fields, names | |
| ) | |
| @dataclass(frozen=True) | |
| class _DataFrameUDTFWrapper: | |
| _func: _UDTFInput | |
| _declaration_context: _UDFDeclarationContext | |
| return_dtype: DataType | |
| _has_named_fields: bool | |
| _deterministic: bool | |
| _name: Optional[str] | |
| @property | |
| def __name__(self) -> str: | |
| return self._create_table_wrapper()._name | |
| def __call__(self, *args: Any) -> _DataFrameUDTFCall: | |
| return _DataFrameUDTFCall( | |
| self._create_table_wrapper()(*args), self.return_dtype, self._has_named_fields | |
| ) | |
| def _create_table_wrapper( | |
| self, input_columns: Optional[Tuple[str, ...]] = None, *, | |
| preserve_field_names: bool = False, | |
| ) -> UserDefinedTableFunctionWrapper: | |
| """Keep Table's mutable row-input flag local to each use of this declaration.""" | |
| result_types = self.return_dtype._to_table_data_type() | |
| if preserve_field_names and isinstance(result_types, RowType): | |
| # Table's RowType path drops field names; its SQL type declaration preserves them. | |
| result_types = _to_java_data_type(result_types).getLogicalType().asSerializableString() | |
| return cast(UserDefinedTableFunctionWrapper, table_udtf( | |
| _DataFrameTableFunctionAdapter( | |
| self._func, | |
| self.return_dtype, | |
| self._deterministic, | |
| input_columns, | |
| ), | |
| result_types=result_types, | |
| deterministic=self._deterministic, | |
| name=self._name, | |
| )) | |
| @overload | |
| def udtf( | |
| func: _UDTFInput, | |
| *, | |
| return_dtype: Optional[_DataTypeLike] = ..., | |
| deterministic: bool = ..., | |
| name: Optional[str] = ..., | |
| ) -> _DataFrameUDTFWrapper: | |
| ... | |
| @overload | |
| def udtf( | |
| func: None = ..., | |
| *, | |
| return_dtype: Optional[_DataTypeLike] = ..., | |
| deterministic: bool = ..., | |
| name: Optional[str] = ..., | |
| ) -> Callable[[_UDTFInput], _DataFrameUDTFWrapper]: | |
| ... | |
| @PublicEvolving() | |
| def udtf( | |
| func: Optional[_UDTFInput] = None, | |
| *, | |
| return_dtype: Optional[_DataTypeLike] = None, | |
| deterministic: bool = True, | |
| name: Optional[str] = None, | |
| ) -> Union[_DataFrameUDTFWrapper, Callable[[_UDTFInput], _DataFrameUDTFWrapper]]: | |
| """ | |
| Declare a synchronous function that emits zero or more rows per invocation. | |
| Supports functions, callable classes and instances, and ``TableFunction`` | |
| classes and instances. Classes must have a zero-argument constructor. Plain | |
| callable classes are instantiated on workers, as with :func:`pyflink.dataframe.udf`. | |
| ``TableFunction`` classes are instantiated on the client when declared; | |
| their ``open`` and ``close`` methods run on workers. Explicitly provided | |
| instances are serialized with the job. | |
| The output type may be explicit or inferred from ``Iterator[T]``, ``Iterable[T]``, | |
| ``Generator[T, ...]``, or ``list[T]``. A ``TypedDict`` supplies output field names. | |
| Return ``None`` for no rows, a list/generator for multiple rows, or a scalar, | |
| tuple, ``Row``, or dict for one row. Nested output values follow scalar UDF rules. | |
| When used with :meth:`DataFrame.flat_map`, the function receives a dictionary | |
| keyed by input column name. :meth:`DataFrame.join_lateral`, expression and SQL | |
| calls pass the specified arguments. Use ``alias`` to name lateral join outputs, | |
| or declare named fields with a ``TypedDict`` or struct ``return_dtype``. | |
| Example:: | |
| >>> from typing import Any, Dict, Iterator, TypedDict | |
| >>> import pyflink.dataframe as pf | |
| >>> class Output(TypedDict): | |
| ... value: int | |
| >>> @pf.udtf | |
| ... def expand(record: Dict[str, Any]) -> Iterator[Output]: | |
| ... yield {"value": record["x"]} | |
| ... yield {"value": record["x"] + 1} | |
| >>> df = pf.from_dict({"x": [1, 2]}) | |
| >>> result = df.flat_map(expand) | |
| Pass a callable class directly for a one-off ``flat_map`` operation. The class | |
| must support zero-argument construction and is instantiated on workers:: | |
| >>> class Repeat: | |
| ... def __init__(self, times=2): | |
| ... self.times = times | |
| ... | |
| ... def __call__(self, record: Dict[str, Any]): | |
| ... for _ in range(self.times): | |
| ... yield record["x"] | |
| >>> repeated = df.flat_map(Repeat, return_dtype=int) | |
| >>> repeated.columns | |
| ['f0'] | |
| Create a UDTF declaration when the callable will be reused or its metadata should | |
| be declared once. Passing the class keeps worker-side construction:: | |
| >>> repeat = pf.udtf(Repeat, return_dtype=int) | |
| >>> first = df.flat_map(repeat) | |
| >>> second = df.flat_map(repeat) | |
| When constructor arguments are needed, create an instance on the client. Pass it | |
| directly for one-off use, or wrap it with :func:`udtf` to reuse the declaration | |
| without repeating its metadata:: | |
| >>> repeat_three_times = Repeat(3) | |
| >>> one_off = df.flat_map(repeat_three_times, return_dtype=int) | |
| >>> reusable_repeat_three_times = pf.udtf(repeat_three_times, return_dtype=int) | |
| >>> reused = df.flat_map(reusable_repeat_three_times) | |
| A ``TableFunction`` class is instantiated on the client when declared. Use an | |
| instance for constructor configuration; in both cases, ``open`` runs on workers:: | |
| >>> from pyflink.table.udf import TableFunction | |
| >>> class Expand(TableFunction): | |
| ... def __init__(self, offset=1): | |
| ... self.offset = offset | |
| ... | |
| ... def open(self, context): | |
| ... self.worker_offset = self.offset | |
| ... | |
| ... def eval(self, record: Dict[str, Any]) -> Iterator[int]: | |
| ... yield record["x"] + self.worker_offset | |
| >>> default_expand = pf.udtf(Expand) | |
| >>> configured_expand = pf.udtf(Expand(offset=10)) | |
| >>> default_result = df.flat_map(default_expand) | |
| >>> configured_result = df.flat_map(configured_expand) | |
| :param func: Function, callable class/instance, or ``TableFunction`` class/instance. | |
| :param return_dtype: Emitted row type, as a DataFrame DataType, Python type, | |
| or SQL type string. Inferred from annotations when omitted. | |
| :param deterministic: Whether equal inputs produce equal results; must agree | |
| with ``TableFunction.is_deterministic``. | |
| :param name: Optional function name. None or an empty string uses the callable's name. | |
| :return: A reusable UDTF declaration, or a decorator when ``func`` is omitted. | |
| .. versionadded:: 2.4.0 | |
| """ | |
| def decorator(f: _UDTFInput) -> _DataFrameUDTFWrapper: | |
| actual_func, context = _resolve_udtf(f) | |
| dtype, has_named_fields = _infer_udtf_return_dtype(context, return_dtype) | |
| if not isinstance(deterministic, bool): | |
| raise TypeError("deterministic must be a bool.") | |
| if isinstance(actual_func, TableFunction): | |
| _validate_determinism_agreement(deterministic, actual_func.is_deterministic()) | |
| return _DataFrameUDTFWrapper( | |
| actual_func, context, dtype, has_named_fields, deterministic, name) | |
| return decorator if func is None else decorator(func) | |
| def _resolve_udtf( | |
| func: _UDTFInput, | |
| ) -> Tuple[_UDTFInput, _UDFDeclarationContext]: | |
| if inspect.isclass(func) and issubclass(func, TableFunction): | |
| _validate_zero_argument_class(func) | |
| func = func() | |
| if not isinstance(func, TableFunction): | |
| raise TypeError("A TableFunction class must construct a TableFunction instance.") | |
| if isinstance(func, TableFunction): | |
| if not callable(func.eval): | |
| raise TypeError("TableFunction.eval must be callable.") | |
| context = _create_declaration_context(func.eval, partial_source=func.eval) | |
| else: | |
| if isinstance(func, UserDefinedFunction) or ( | |
| inspect.isclass(func) and issubclass(func, UserDefinedFunction) | |
| ): | |
| raise TypeError("func must be a table UDF or a Python callable.") | |
| context = _resolve_udf(func).declaration_context | |
| target = context.annotation_target | |
| try: | |
| unwrapped_target = inspect.unwrap(target) | |
| except ValueError as exc: | |
| raise TypeError("Cannot inspect a UDTF with a wrapper cycle.") from exc | |
| if any( | |
| inspect.iscoroutinefunction(candidate) or inspect.isasyncgenfunction(candidate) | |
| for candidate in (target, unwrapped_target) | |
| ): | |
| raise TypeError("DataFrame UDTFs must be synchronous; async functions are not supported.") | |
| return func, context | |
| def _infer_udtf_return_dtype( | |
| context: _UDFDeclarationContext, return_dtype: Optional[_DataTypeLike] | |
| ) -> Tuple[DataType, bool]: | |
| if return_dtype is not None: | |
| dtype = _convert_to_dtype(return_dtype) | |
| has_named_fields = isinstance(dtype._to_table_data_type(), RowType) | |
| else: | |
| hint = _get_callable_return_type_hint(context) | |
| origin, arguments = get_origin(hint), get_args(hint) | |
| if origin in ( | |
| collections.abc.Iterator, collections.abc.Iterable, collections.abc.Generator, list | |
| ) and arguments: | |
| hint = arguments[0] | |
| elif hint is _UNRESOLVED_TYPE_HINT or origin in ( | |
| collections.abc.Iterator, collections.abc.Iterable, collections.abc.Generator, list | |
| ): | |
| raise TypeError("Cannot infer UDTF return type; specify return_dtype explicitly.") | |
| if get_origin(hint) is tuple: | |
| fields = get_args(hint) | |
| if not fields or Ellipsis in fields: | |
| raise TypeError("UDTF tuple outputs must have a fixed number of fields.") | |
| dtype = DataType.struct([ | |
| (f"f{index}", DataType._from_type_hint(field)) | |
| for index, field in enumerate(fields) | |
| ]) | |
| has_named_fields = False | |
| else: | |
| dtype = DataType._from_type_hint(hint) | |
| has_named_fields = isinstance(dtype._to_table_data_type(), RowType) | |
| table_type = dtype._to_table_data_type() | |
| if isinstance(table_type, RowType) and not table_type.fields: | |
| raise ValueError("A UDTF must declare at least one output field.") | |
| return dtype, has_named_fields | |
| def _validate_flat_map_input(declaration: _DataFrameUDTFWrapper) -> None: | |
| source = declaration._func | |
| target: Callable[..., Any] | |
| if inspect.isclass(source): | |
| target = declaration._declaration_context.annotation_target | |
| elif isinstance(source, TableFunction): | |
| target = source.eval | |
| elif isinstance(source, functools.partial): | |
| target = source | |
| else: | |
| target = _get_callable_inspection_target(source) | |
| if inspect.ismethod(target) and hasattr(target, "__wrapped__") and not ( | |
| _preserves_method_binding(target, declaration._declaration_context.defining_class) | |
| ): | |
| target = target.__func__ | |
| try: | |
| signature = inspect.signature(target) | |
| except (TypeError, ValueError): | |
| return | |
| if inspect.isclass(source): | |
| descriptor = inspect.getattr_static(source, "__call__") | |
| binds_receiver = not isinstance(descriptor, staticmethod) | |
| if hasattr(target, "__wrapped__"): | |
| binds_receiver = binds_receiver and _preserves_method_binding( | |
| target, declaration._declaration_context.defining_class) | |
| parameters = list(signature.parameters.values()) | |
| if binds_receiver and parameters and parameters[0].kind in ( | |
| inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD | |
| ): | |
| signature = signature.replace(parameters=parameters[1:]) | |
| parameters = list(signature.parameters.values()) | |
| try: | |
| signature.bind(object()) | |
| except TypeError as exc: | |
| raise ValueError("flat_map requires a function accepting one row argument.") from exc | |
| positional = [p for p in parameters if p.kind in ( | |
| inspect.Parameter.POSITIONAL_ONLY, inspect.Parameter.POSITIONAL_OR_KEYWORD | |
| )] | |
| if not positional: | |
| return | |
| hint = _resolve_callable_annotation(declaration._declaration_context, positional[0].name) | |
| if not _is_flat_map_row_hint(hint): | |
| raise ValueError(f"flat_map receives one dict row argument, got annotation {hint}.") | |
| def _is_flat_map_row_hint(hint: Any) -> bool: | |
| if hint in (_UNRESOLVED_TYPE_HINT, Any, object): | |
| return True | |
| origin = get_origin(hint) | |
| if origin is Annotated: | |
| return _is_flat_map_row_hint(get_args(hint)[0]) | |
| if origin in (Union, getattr(types, "UnionType", Union)): | |
| return any(_is_flat_map_row_hint(member) for member in get_args(hint)) | |
| if _is_typed_dict(hint): | |
| return True | |
| target = origin or hint | |
| try: | |
| return issubclass(dict, target) | |
| except TypeError: | |
| # Leave annotations that cannot be checked at runtime to the user's type checker. | |
| return True | |
| def _resolve_flat_map_udtf( | |
| func: Union[Callable[..., Any], Type, _DataFrameUDTFWrapper], | |
| return_dtype: Optional[_DataTypeLike], | |
| input_columns: List[str], | |
| ) -> Tuple[Expression, List[str]]: | |
| if isinstance(func, _DataFrameUDTFWrapper): | |
| if return_dtype is not None: | |
| raise ValueError("return_dtype must not be specified for a DataFrame UDTF declaration.") | |
| declaration = func | |
| else: | |
| if func is None: | |
| raise TypeError("flat_map requires a callable or a pf.udtf declaration.") | |
| if isinstance(func, UserDefinedFunction) or ( | |
| inspect.isclass(func) and issubclass(func, UserDefinedFunction) | |
| ): | |
| raise TypeError("flat_map accepts Python callables or a pf.udtf declaration.") | |
| declaration = udtf(func, return_dtype=return_dtype) | |
| _validate_flat_map_input(declaration) | |
| table_type = declaration.return_dtype._to_table_data_type() | |
| if declaration._has_named_fields: | |
| output_columns = cast(RowType, table_type).field_names() | |
| elif isinstance(table_type, RowType) and len(table_type.fields) > 1: | |
| raise ValueError("flat_map requires named output fields; use TypedDict or a named struct.") | |
| else: | |
| output_columns = ["f0"] | |
| wrapper = declaration._create_table_wrapper(tuple(input_columns)) | |
| wrapper._set_takes_row_as_input() | |
| return wrapper(with_columns(col("*"))), output_columns | |
| def _resolve_lateral_expression( | |
| call: Union[_DataFrameUDTFCall, Expression], input_columns: List[str], | |
| ) -> Expression: | |
| if isinstance(call, _DataFrameUDTFCall): | |
| expression = call.expression | |
| if call.output_aliases is not None: | |
| output_names = list(call.output_aliases) | |
| elif call.has_named_fields: | |
| output_names = cast(RowType, call.return_dtype._to_table_data_type()).field_names() | |
| expression = expression.alias(*output_names) | |
| else: | |
| raise ValueError( | |
| "UDTF calls passed to join_lateral require output names; use alias, " | |
| "a TypedDict, or a named struct return_dtype.") | |
| elif isinstance(call, Expression): | |
| expression = call | |
| gateway = get_gateway() | |
| api_utils = gateway.jvm.org.apache.flink.table.expressions.ApiExpressionUtils | |
| definitions = gateway.jvm.org.apache.flink.table.functions.BuiltInFunctionDefinitions | |
| j_expression = api_utils.unwrapFromApi(expression._j_expr) | |
| if not api_utils.isFunction(j_expression, definitions.AS): | |
| raise ValueError("Expression arguments passed to join_lateral require alias.") | |
| string_class = get_java_class(gateway.jvm.String) | |
| output_names = [ | |
| child.getValueAs(string_class).get() for child in list(j_expression.getChildren())[1:] | |
| ] | |
| else: | |
| raise TypeError( | |
| "table_function_call must be a DataFrame UDTF call or Expression, " | |
| f"got {type(call).__name__}") | |
| if not output_names or not all(output_names) or len(set(output_names)) != len(output_names): | |
| raise ValueError("join_lateral output names must be non-empty and unique.") | |
| conflicts = set(input_columns).intersection(output_names) | |
| if conflicts: | |
| raise ValueError( | |
| f"join_lateral output names conflict with input columns: {sorted(conflicts)}") | |
| return expression | |
| def _iter_user_results(result: Any) -> Iterator[Any]: | |
| if result is None: | |
| return | |
| if isinstance(result, (Row, tuple, collections.abc.Mapping, str, bytes, bytearray)): | |
| yield result | |
| elif isinstance(result, collections.abc.Iterable): | |
| yield from result | |
| else: | |
| yield result | |
| class _DataFrameTableFunctionAdapter(TableFunction): | |
| def __init__( | |
| self, | |
| func: _UDTFInput, | |
| return_dtype: DataType, | |
| deterministic: bool, | |
| input_columns: Optional[Tuple[str, ...]], | |
| ) -> None: | |
| self._func = func | |
| self._return_dtype = return_dtype | |
| self._deterministic = deterministic | |
| self._input_columns = input_columns | |
| self._bound_invocation: Optional[Callable[..., Any]] = None | |
| self._lifecycle_opened = False | |
| self.__name__ = getattr(func, "__name__", type(func).__name__) | |
| def open(self, function_context: Any) -> None: | |
| if isinstance(self._func, TableFunction): | |
| self._func.open(function_context) | |
| self._lifecycle_opened = True | |
| invoke_func = self._func.eval | |
| else: | |
| invoke_func = self._func() if inspect.isclass(self._func) else self._func | |
| if not callable(invoke_func): | |
| raise TypeError("UDTF class must construct a callable instance.") | |
| try: | |
| self._bound_invocation = self._bind_func(invoke_func) | |
| except Exception: | |
| try: | |
| self.close() | |
| except Exception: | |
| pass | |
| raise | |
| def close(self) -> None: | |
| try: | |
| if self._lifecycle_opened: | |
| cast(TableFunction, self._func).close() | |
| finally: | |
| self._bound_invocation = None | |
| self._lifecycle_opened = False | |
| def is_deterministic(self) -> bool: | |
| if isinstance(self._func, TableFunction): | |
| return self._func.is_deterministic() | |
| return self._deterministic | |
| def _bind_func(self, invoke_func: Callable[..., Any]) -> Callable[..., Any]: | |
| table_type = self._return_dtype._to_table_data_type() | |
| normalizer = _create_result_normalizer(table_type) | |
| is_struct = isinstance(table_type, RowType) | |
| field_count = len(table_type.fields) if isinstance(table_type, RowType) else 1 | |
| input_columns = list(self._input_columns) if self._input_columns is not None else None | |
| def invoke(*args: Any) -> Iterator[Row]: | |
| if input_columns is not None: | |
| input_row = args[0] | |
| row = {name: input_row[i] for i, name in enumerate(input_columns)} | |
| result = invoke_func(row) | |
| else: | |
| result = invoke_func(*args) | |
| for item in _iter_user_results(result): | |
| if is_struct and field_count == 1 and item is not None: | |
| if not isinstance(item, (Row, tuple, collections.abc.Mapping)): | |
| item = Row(item) | |
| if not is_struct and isinstance(item, (Row, tuple)): | |
| if len(item) != 1: | |
| raise ValueError("UDTF scalar output requires exactly one field.") | |
| item = item[0] | |
| normalized = normalizer(item) if normalizer is not None else item | |
| if is_struct: | |
| if normalized is None: | |
| yield Row(*([None] * field_count)) | |
| else: | |
| yield normalized | |
| else: | |
| yield Row(normalized) | |
| return invoke | |
| def eval(self, *args: Any) -> Iterator[Row]: | |
| if self._bound_invocation is None: | |
| raise RuntimeError("DataFrame UDTF was invoked before open().") | |
| return self._bound_invocation(*args) |