blob: dd82b267f6fc4c70199175af63f00d38aab26686 [file]
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
"""User-defined aggregate functions for the DataFrame API."""
import inspect
from dataclasses import dataclass, field
from typing import Any, Callable, Optional, Tuple, Type, Union, cast, overload
from pyflink.dataframe.datatype import DataType
from pyflink.dataframe.udf import (
_DataTypeLike,
_UDFDeclarationContext,
_convert_to_dtype,
_create_declaration_context,
_create_result_normalizer,
_infer_return_dtype,
_resolve_udf,
_validate_determinism_agreement,
_validate_zero_argument_class,
)
from pyflink.table.expression import Expression
from pyflink.table.types import DataType as TableDataType, MapType
from pyflink.table.udf import (
AggregateFunction,
DelegatingPandasAggregateFunction,
UserDefinedAggregateFunctionWrapper,
UserDefinedFunction,
udaf as table_udaf,
)
from pyflink.util.api_stability_decorators import PublicEvolving
__all__ = ["udaf"]
_UDAFInput = Union[Callable[..., Any], AggregateFunction, Type]
@dataclass(frozen=True)
class _DataFrameUDAFWrapper:
_func: _UDAFInput
return_dtype: DataType
accumulator_type: Optional[DataType]
_func_type: str
_deterministic: bool
_name: Optional[str]
_cached_table_udf_wrapper: Optional[UserDefinedAggregateFunctionWrapper] = field(
default=None, init=False, repr=False, compare=False)
@property
def __name__(self) -> str:
return self._table_udf_wrapper._name
def __call__(self, *args: Any) -> Expression:
return self._table_udf_wrapper(*args)
@property
def _table_udf_wrapper(self) -> UserDefinedAggregateFunctionWrapper:
if self._cached_table_udf_wrapper is None:
wrapper = table_udaf(
_DataFrameAggregateFunctionAdapter(
self._func, self.return_dtype, self._deterministic, self._func_type),
result_type=self.return_dtype._to_table_data_type(),
accumulator_type=(self.accumulator_type._to_table_data_type()
if self.accumulator_type is not None else None),
func_type=self._func_type,
deterministic=self._deterministic,
name=self._name,
)
object.__setattr__(self, "_cached_table_udf_wrapper", wrapper)
return cast(UserDefinedAggregateFunctionWrapper, self._cached_table_udf_wrapper)
@overload
def udaf(
func: _UDAFInput, *, return_dtype: Optional[_DataTypeLike] = ...,
accumulator_type: Optional[_DataTypeLike] = ..., func_type: str = ...,
deterministic: bool = ..., name: Optional[str] = ...,
) -> _DataFrameUDAFWrapper:
...
@overload
def udaf(
func: None = ..., *, return_dtype: Optional[_DataTypeLike] = ...,
accumulator_type: Optional[_DataTypeLike] = ..., func_type: str = ...,
deterministic: bool = ..., name: Optional[str] = ...,
) -> Callable[[_UDAFInput], _DataFrameUDAFWrapper]:
...
@PublicEvolving()
def udaf(
func: Optional[_UDAFInput] = None,
*,
return_dtype: Optional[_DataTypeLike] = None,
accumulator_type: Optional[_DataTypeLike] = None,
func_type: str = "general",
deterministic: bool = True,
name: Optional[str] = None,
) -> Union[_DataFrameUDAFWrapper, Callable[[_UDAFInput], _DataFrameUDAFWrapper]]:
"""
Declare a function that aggregates multiple rows into one result.
General UDAFs require an :class:`~pyflink.table.udf.AggregateFunction` instance
or class. Pandas UDAFs also accept functions and callable instances or classes;
each argument contains a group or window's column values as pandas data.
Calling the declaration with column expressions produces an aggregation expression
for ``agg()``. Declarations can also be bound in :func:`pyflink.dataframe.sql`.
``AggregateFunction`` classes are constructed on the client. Plain callable classes
are constructed on workers. Classes must have a zero-argument constructor;
provided instances are serialized with the job. ``AggregateFunction.open`` and
``AggregateFunction.close`` run on workers.
Explicit types take precedence over ``get_result_type()`` and
``get_accumulator_type()``. Without a result type, the return annotation of
``get_value()`` or the callable is used. Accumulator types are not inferred.
Aggregation results follow the nested value conversion rules of DataFrame scalar UDFs.
Ordinary grouped and global aggregations support general UDAFs in streaming mode
and pandas UDAFs in batch mode. Window aggregations follow Table API support.
Example::
>>> import pyflink.dataframe as pf
>>> from pyflink.table import EnvironmentSettings, TableEnvironment
>>> from pyflink.table.udf import AggregateFunction
>>> pf.set_table_environment(TableEnvironment.create(
... EnvironmentSettings.in_streaming_mode()))
>>> class Sum(AggregateFunction):
... def create_accumulator(self):
... return [0]
... def accumulate(self, accumulator, value):
... accumulator[0] += value
... def get_value(self, accumulator) -> int:
... return accumulator[0]
>>> total = pf.udaf(Sum, accumulator_type=list[int])
>>> df = pf.from_dict({"category": ["a", "a", "b"], "value": [1, 2, 3]})
>>> result = df.group_by("category").agg(total=total(pf.col("value")))
>>> result.columns
['category', 'total']
A pandas callable can be used in batch aggregations::
>>> pf.set_table_environment(TableEnvironment.create(
... EnvironmentSettings.in_batch_mode()))
>>> df = pf.from_dict({"category": ["a", "a", "b"], "value": [1, 2, 3]})
>>> @pf.udaf(return_dtype=float, func_type="pandas")
... def mean(values):
... return values.mean()
>>> result = df.group_by("category").agg(mean=mean(pf.col("value")))
>>> result = pf.sql(
... "SELECT category, avg_value(`value`) AS mean FROM src GROUP BY category",
... auto_bind=False, src=df, avg_value=mean)
:param func: AggregateFunction instance/class, or a pandas callable instance/class/function.
:param return_dtype: Result type as a DataFrame DataType, Python type, or SQL type string.
:param accumulator_type: General UDAF state type, in the same formats as ``return_dtype``.
Omit when ``get_accumulator_type()`` supplies it. Pandas UDAFs
use the accumulator type managed by the Table API.
:param func_type: ``"general"`` (default) or ``"pandas"``.
:param deterministic: Must agree with ``AggregateFunction.is_deterministic()``.
:param name: Optional function name. None or an empty string uses the callable's name.
:return: A reusable aggregate declaration, or a decorator when ``func`` is omitted.
.. versionadded:: 2.4.0
"""
if func_type not in ("general", "pandas"):
raise ValueError("func_type must be 'general' or 'pandas'.")
if not isinstance(deterministic, bool):
raise TypeError("deterministic must be a bool.")
def decorator(f: _UDAFInput) -> _DataFrameUDAFWrapper:
actual_func, context = _resolve_udaf(f, func_type)
result = _resolve_aggregate_type(actual_func, return_dtype, "get_result_type")
if result is None:
result = _infer_return_dtype(context, None, name or "UDAF")
if func_type == "pandas" and isinstance(result._to_table_data_type(), MapType):
raise TypeError("Pandas UDAFs do not support MAP results.")
accumulator = _resolve_aggregate_type(
actual_func, accumulator_type, "get_accumulator_type"
) if func_type == "general" or accumulator_type is not None else None
if func_type == "general" and accumulator is None:
raise TypeError("Specify accumulator_type or implement get_accumulator_type().")
if isinstance(actual_func, AggregateFunction):
_validate_determinism_agreement(deterministic, actual_func.is_deterministic())
return _DataFrameUDAFWrapper(
actual_func, result, accumulator, func_type, deterministic, name)
return decorator if func is None else decorator(func)
def _resolve_udaf(func: _UDAFInput, func_type: str) -> Tuple[_UDAFInput, _UDFDeclarationContext]:
if inspect.isclass(func) and issubclass(func, AggregateFunction):
_validate_zero_argument_class(func)
func = func()
if not isinstance(func, AggregateFunction):
raise TypeError(
"An AggregateFunction class must construct an AggregateFunction instance.")
if isinstance(func, AggregateFunction):
for method_name in (
"create_accumulator", "accumulate", "get_value", "retract", "merge",
"open", "close", "get_result_type", "get_accumulator_type", "is_deterministic",
):
_validate_sync_aggregate_method(getattr(func, method_name))
context = _create_declaration_context(func.get_value, partial_source=func.get_value)
else:
if func_type == "general":
raise TypeError("General UDAFs require an AggregateFunction instance or class.")
if isinstance(func, UserDefinedFunction) or (
inspect.isclass(func) and issubclass(func, UserDefinedFunction)
):
raise TypeError("Pandas UDAFs require an AggregateFunction or a Python callable.")
context = _resolve_udf(func).declaration_context
_validate_sync_aggregate_method(context.annotation_target)
return func, context
def _validate_sync_aggregate_method(func: Callable[..., Any]) -> None:
if not callable(func):
raise TypeError("UDAF methods must be callable.")
for candidate in (func, inspect.unwrap(func)):
if inspect.iscoroutinefunction(candidate) or inspect.isasyncgenfunction(candidate):
raise TypeError("UDAFs must be synchronous; async functions are not supported.")
def _resolve_aggregate_type(
func: _UDAFInput, explicit_type: Optional[_DataTypeLike], method_name: str,
) -> Optional[DataType]:
if explicit_type is not None:
return _convert_to_dtype(explicit_type)
if isinstance(func, AggregateFunction):
method = getattr(func, method_name)
if getattr(method, "__func__", None) is not getattr(AggregateFunction, method_name):
declared_type = method()
if isinstance(declared_type, TableDataType):
return DataType(declared_type)
if isinstance(declared_type, str):
return _convert_to_dtype(declared_type)
raise TypeError(f"{method_name}() must return a Table DataType or SQL type string.")
return None
class _DataFrameAggregateFunctionAdapter(AggregateFunction):
def __init__(
self, func: _UDAFInput, return_dtype: DataType, deterministic: bool, func_type: str,
) -> None:
self._func = func
self._return_dtype = return_dtype
self._deterministic = deterministic
self._func_type = func_type
self._active_func: Optional[AggregateFunction] = None
self._normalizer: Optional[Callable[[Any], Any]] = None
self.__name__ = getattr(func, "__name__", type(func).__name__)
def open(self, function_context: Any) -> None:
if isinstance(self._func, AggregateFunction):
active_func = self._func
else:
invoke = self._func() if inspect.isclass(self._func) else self._func
if not callable(invoke):
raise TypeError("UDAF class must construct a callable instance.")
active_func = DelegatingPandasAggregateFunction(invoke)
# Arrow requires tuples or dictionaries for structs, including nested ones.
normalizer = _create_result_normalizer(
self._return_dtype._to_table_data_type(), row_as_tuple=self._func_type == "pandas")
active_func.open(function_context)
self._active_func = active_func
self._normalizer = normalizer
def close(self) -> None:
try:
if self._active_func is not None:
self._active_func.close()
finally:
self._active_func = None
self._normalizer = None
def is_deterministic(self) -> bool:
if isinstance(self._func, AggregateFunction):
return self._func.is_deterministic()
return self._deterministic
def _active(self) -> AggregateFunction:
if self._active_func is None:
raise RuntimeError("UDAF was invoked before open().")
return self._active_func
def create_accumulator(self) -> Any:
return self._active().create_accumulator()
def accumulate(self, accumulator: Any, *args: Any) -> None:
self._active().accumulate(accumulator, *args)
def retract(self, accumulator: Any, *args: Any) -> None:
self._active().retract(accumulator, *args)
def merge(self, accumulator: Any, accumulators: Any) -> None:
self._active().merge(accumulator, accumulators)
def get_value(self, accumulator: Any) -> Any:
result: Any = self._active().get_value(accumulator)
return self._normalizer(result) if self._normalizer is not None else result