blob: f423a8990ee0ef53fe60ae3fe7196856b3486280 [file]
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
import asyncio
import functools
import importlib
import inspect
import operator
import types
import unittest
from dataclasses import dataclass
from typing import Any, Callable, Optional, TypedDict, cast
from unittest import mock
import pandas as pd
import pyarrow as pa
import pyflink.dataframe as pf
from pyflink.common import Row, RowKind
from pyflink.table import DataTypes as TableDataTypes
from pyflink.table.expression import Expression
from pyflink.table.types import MapType, RowType
from pyflink.table.udf import AsyncScalarFunction, ScalarFunction, TableFunction
from pyflink.testing.test_case_utils import (
PyFlinkDataFrameUTTestCase,
PyFlinkStreamDataFrameTestCase,
)
def _return_dtype(declaration: Callable[..., Expression]) -> pf.DataType:
return cast(Any, declaration).return_dtype
_UDF_TEST_ALIAS = int
def _module_alias_method(self, value: int) -> "_UDF_TEST_ALIAS":
return value
def _module_alias_function(value: int) -> "_UDF_TEST_ALIAS":
return value
def _call_module_alias_function(value):
return _module_alias_function(value)
class DataFrameUDFDeclarationTests(unittest.TestCase):
def test_arrow_annotation_inference_and_overrides(self):
def arrow_identity(values: pa.Array) -> pa.ChunkedArray:
return pa.chunked_array([values])
def integer_array(values: pa.Int64Array) -> pa.Int64Array:
return values
def list_array(values: pa.ListArray):
return values.flatten()
def struct_array(values: "pa.StructArray"):
return values.field("value")
def concrete_return(values) -> pa.Int64Array:
return values
def mixed(values: pd.Series) -> pa.Int64Array:
return pa.array(values)
def captured(context: pd.Series, values: pa.Array) -> pa.Array:
return values
class ArrowCallable:
def __call__(self, values: "pa.Array") -> "pa.Array":
return values
class ArrowScalar(ScalarFunction):
def eval(self, values: pa.Array) -> pa.Array:
return values
for func in (arrow_identity, integer_array, list_array, struct_array, concrete_return,
ArrowCallable, ArrowCallable(), ArrowScalar, ArrowScalar(),
functools.partial(captured, pd.Series([1]))):
with self.subTest(func=func):
declaration = pf.udf(func, return_dtype=pf.DataType.int64())
self.assertEqual(cast(Any, declaration)._func_type, "arrow")
for mode in ("general", "pandas", "arrow"):
with self.subTest(mode=mode):
declaration = pf.udf(mixed, return_dtype=pf.DataType.int64(), func_type=mode)
self.assertEqual(cast(Any, declaration)._func_type, mode)
with self.assertRaisesRegex(ValueError, "pandas.*Arrow.*func_type"):
pf.udf(mixed, return_dtype=pf.DataType.int64())
with self.assertRaisesRegex(TypeError, "return_dtype is required for arrow"):
pf.udf(arrow_identity)
def test_explicit_arrow_declarations(self):
from pyflink.table.udf import udf as table_udf
def identity(values):
return values
declaration = pf.udf(identity, return_dtype=pf.DataType.string(), func_type="arrow")
self.assertEqual(_return_dtype(declaration), pf.DataType.string())
table_udf(identity, result_type=TableDataTypes.STRING(), func_type="arrow")
with self.assertRaisesRegex(TypeError, "return_dtype is required for arrow"):
pf.udf(identity, func_type="arrow")
async def async_identity(values):
return values
class AsyncCallable:
async def __call__(self, values):
return values
for declare in (
lambda: pf.udf(async_identity, return_dtype=pf.DataType.string(), func_type="arrow"),
lambda: table_udf(async_identity, result_type=TableDataTypes.STRING(),
func_type="arrow"),
lambda: table_udf(AsyncCallable(), result_type=TableDataTypes.STRING(),
func_type="arrow"),
lambda: table_udf(functools.partial(AsyncCallable()),
result_type=TableDataTypes.STRING(), func_type="arrow"),
):
with self.subTest(declare=declare):
with self.assertRaisesRegex(ValueError, "Async.*arrow"):
declare()
def test_function_declarations_return_types_and_metadata(self):
class Details(TypedDict):
label: str
scores: list[int]
class Result(TypedDict):
id: int
details: Details
def add_one(value: int) -> int:
"""Add one to a value."""
return value + 1
def optional_return(value: int) -> Optional[int]:
return value if value % 2 == 0 else None
def identity(value):
return value
def describe(value: int) -> Result:
return {
"id": value,
"details": {"label": str(value), "scores": [value]},
}
def concrete_return_with_unresolved_input(value):
return value
concrete_return_with_unresolved_input.__annotations__ = {
"value": "UnavailableInput",
"return": int,
}
def postponed_return_with_unresolved_input(value):
return value
postponed_return_with_unresolved_input.__annotations__ = {
"value": "UnavailableInput",
"return": "int",
}
decorated: Callable[..., Expression] = pf.udf(add_one)
self.assertFalse(hasattr(pf, "DataFrameUDFWrapper"))
udf_module = importlib.import_module("pyflink.dataframe.udf")
self.assertFalse(hasattr(udf_module, "DataFrameUDFWrapper"))
self.assertEqual(_return_dtype(decorated), pf.DataType.int64().not_null())
self.assertEqual(decorated.__name__, "add_one")
self.assertEqual(decorated.__qualname__, add_one.__qualname__)
self.assertEqual(decorated.__module__, add_one.__module__)
self.assertEqual(decorated.__doc__, "Add one to a value.")
self.assertNotIn("__wrapped__", vars(decorated))
self.assertNotIn("__signature__", vars(decorated))
self.assertNotIn("__annotations__", vars(decorated))
wrapper_signature = inspect.signature(decorated)
parameters = tuple(wrapper_signature.parameters.values())
self.assertEqual(len(parameters), 1)
self.assertEqual(parameters[0].name, "args")
self.assertIs(parameters[0].kind, inspect.Parameter.VAR_POSITIONAL)
self.assertIs(parameters[0].annotation, Any)
self.assertIs(wrapper_signature.return_annotation, Expression)
configured: Callable[..., Expression] = pf.udf(
return_dtype=pf.DataType.string()
)(
lambda value: str(value)
)
direct = pf.udf(functools.partial(add_one), name="partial_add_one")
self.assertEqual(_return_dtype(configured), pf.DataType.string())
self.assertEqual(_return_dtype(direct), pf.DataType.int64().not_null())
self.assertEqual(direct.__name__, "partial_add_one")
with mock.patch.object(
pf.DataType,
"_from_sql",
return_value=pf.DataType.int64(),
) as from_sql:
sql_typed = pf.udf(identity, return_dtype="BIGINT")
self.assertEqual(_return_dtype(sql_typed), pf.DataType.int64())
from_sql.assert_called_once_with("BIGINT")
expected_result_dtype = pf.DataType.struct(
{
"id": pf.DataType.int64().not_null(),
"details": pf.DataType.struct(
{
"label": pf.DataType.string().not_null(),
"scores": pf.DataType.list(
pf.DataType.int64().not_null()
).not_null(),
}
).not_null(),
}
).not_null()
declarations = [
(
"Python type",
lambda: pf.udf(identity, return_dtype=int),
pf.DataType.int64().not_null(),
),
(
"Optional return annotation widens to nullable",
lambda: pf.udf(optional_return),
pf.DataType.int64(),
),
(
"nested TypedDict annotation",
lambda: pf.udf(describe),
expected_result_dtype,
),
(
"explicit nested TypedDict",
lambda: pf.udf(identity, return_dtype=Result),
expected_result_dtype,
),
(
"concrete return with unresolved input",
lambda: pf.udf(concrete_return_with_unresolved_input),
pf.DataType.int64().not_null(),
),
(
"postponed return with unresolved input",
lambda: pf.udf(postponed_return_with_unresolved_input),
pf.DataType.int64().not_null(),
),
]
for case_name, declare, expected in declarations:
with self.subTest(case=case_name):
self.assertEqual(_return_dtype(declare()), expected)
def test_callable_classes_and_instances_infer_from_invocation_method(self):
plain_constructor_calls = []
scalar_constructor_calls = []
class AddOne:
def __init__(self):
plain_constructor_calls.append("AddOne")
def __call__(self, value: int) -> int:
return value + 1
class AddOffset:
def __init__(self, offset):
self.offset = offset
def __call__(self, value: int) -> int:
return value + self.offset
class NamedCallable:
__name__ = "configured_add"
def __call__(self, value: int) -> int:
return value + 1
class Double(ScalarFunction):
def __init__(self):
scalar_constructor_calls.append("Double")
def eval(self, *values: int) -> int:
value, = values
return value * 2
class AsyncDouble(AsyncScalarFunction):
def __init__(self):
scalar_constructor_calls.append("AsyncDouble")
async def eval(self, *values: int) -> int:
value, = values
return value * 2
class AddScalarOffset(ScalarFunction):
def __init__(self, offset):
self.offset = offset
def eval(self, *values: int) -> int:
value, = values
return value + self.offset
named_callable = NamedCallable()
double_instance = Double()
async_double_instance = AsyncDouble()
scalar_constructor_calls.clear()
callables = [
AddOne,
AddOffset(2),
named_callable,
Double,
double_instance,
AsyncDouble,
async_double_instance,
AddScalarOffset(2),
]
for source in callables:
with self.subTest(source=source):
decorated = pf.udf(source)
self.assertEqual(
_return_dtype(decorated), pf.DataType.int64().not_null()
)
self.assertEqual(plain_constructor_calls, [])
self.assertEqual(scalar_constructor_calls, [])
self.assertEqual(pf.udf(named_callable).__name__, "configured_add")
decorated_class = pf.udf(Double)
self.assertEqual(decorated_class.__qualname__, Double.__qualname__)
def test_callable_class_resolves_class_local_return_annotation(self):
class Describe:
class Output(TypedDict):
value: int
def __call__(self, value: int) -> "Output":
return {"value": value}
expected = pf.DataType.struct({"value": pf.DataType.int64().not_null()}).not_null()
for source in (Describe, Describe()):
with self.subTest(source=source):
self.assertEqual(_return_dtype(pf.udf(source)), expected)
def test_callable_annotations_use_lexical_defining_class(self):
class BoundMethodOwner:
class Output(TypedDict):
value: int
def describe(self, value: int) -> "Output":
return {"value": value}
class InheritedMethodOwner:
class Output(TypedDict):
value: int
def __call__(self, value: int) -> "Output":
return {"value": value}
class InheritedCallable(InheritedMethodOwner):
pass
class SelfQualified:
class Output(TypedDict):
value: int
def __call__(self, value: int) -> "SelfQualified.Output":
return {"value": value}
expected = pf.DataType.struct({"value": pf.DataType.int64().not_null()}).not_null()
bound_method = BoundMethodOwner().describe
for source in (
bound_method,
functools.partial(bound_method),
InheritedCallable,
SelfQualified,
):
with self.subTest(source=source):
self.assertEqual(_return_dtype(pf.udf(source)), expected)
class PandasCallable:
Batch = pd.Series
def __call__(self, values: "Batch") -> int:
return len(values)
pandas_declaration = pf.udf(PandasCallable, return_dtype=int)
self.assertEqual(pandas_declaration._func_type, "pandas")
class ReceivingCallable:
_UDF_TEST_ALIAS = str
__call__ = _module_alias_method
self.assertEqual(
_return_dtype(pf.udf(ReceivingCallable)),
pf.DataType.int64().not_null(),
)
class OverriddenScalarFunction(ScalarFunction):
_UDF_TEST_ALIAS = str
def eval(self, *values: int) -> str:
value, = values
return str(value)
overridden = OverriddenScalarFunction()
overridden.eval = types.MethodType(_module_alias_method, overridden)
self.assertEqual(
_return_dtype(pf.udf(overridden)), pf.DataType.int64().not_null()
)
def test_wrapped_callable_annotations_and_partial_validation(self):
def add(value: int, amount: int = 1) -> int:
return value + amount
def pandas_identity(values: pd.Series) -> pd.Series:
return values
class WrappedCallableClass:
@functools.wraps(pandas_identity)
def __call__(self, *args, **kwargs):
return pandas_identity(*args, **kwargs)
class WrappedClassMethodCallableClass:
@classmethod
@functools.wraps(pandas_identity)
def __call__(cls, *args, **kwargs):
return pandas_identity(*args, **kwargs)
class WrappedScalarFunction(ScalarFunction):
@functools.wraps(pandas_identity)
def eval(self, *args, **kwargs):
return pandas_identity(*args, **kwargs)
with self.assertRaisesRegex(
TypeError, "Invalid functools.partial UDF 'add'.*unexpected keyword"
):
pf.udf(functools.partial(add, missing=1))
uninspectable = pf.udf(operator.itemgetter(0), return_dtype=int)
self.assertEqual(
_return_dtype(uninspectable), pf.DataType.int64().not_null()
)
wrapped_callable_instance = WrappedCallableClass()
for source in (
WrappedCallableClass,
wrapped_callable_instance,
wrapped_callable_instance.__call__,
WrappedClassMethodCallableClass,
WrappedClassMethodCallableClass(),
WrappedScalarFunction,
WrappedScalarFunction(),
):
with self.subTest(wrapped_source=source):
declaration = pf.udf(
source, return_dtype=pf.DataType.int64()
)
self.assertEqual(declaration._func_type, "pandas")
cross_namespace_wrapper = types.FunctionType(
_call_module_alias_function.__code__,
{
"_module_alias_function": _module_alias_function,
"_UDF_TEST_ALIAS": str,
},
)
functools.update_wrapper(cross_namespace_wrapper, _module_alias_function)
self.assertEqual(
_return_dtype(pf.udf(cross_namespace_wrapper)),
pf.DataType.int64().not_null(),
)
def test_func_type_resolution_and_async_detection(self):
def method_decorator(method):
@functools.wraps(method)
def wrapper(*args, **kwargs):
return method(*args, **kwargs)
return wrapper
def pandas_add_one(values: pd.Series) -> pd.Series:
return values + 1
def with_pandas_context(context: pd.Series, value: int) -> int:
return value
def pandas_forward_reference(values):
return values
pandas_forward_reference.__annotations__["values"] = "pandas.Series"
def pandas_with_unresolved_annotation(
values: pd.Series, context
) -> pd.Series:
return values
pandas_with_unresolved_annotation.__annotations__[
"context"
] = "UnavailableContext"
def pandas_after_missing_attribute(
context: Any, values: pd.Series
) -> int:
return len(values)
pandas_after_missing_attribute.__annotations__[
"context"
] = "pd.Missing"
def only_missing_attribute(context: Any) -> int:
return 1
only_missing_attribute.__annotations__["context"] = "pd.Missing"
def mixed(values: pd.Series, offset: int):
return values + offset
def arrow_add_one(values: pa.Array) -> pa.Array:
return pa.array([value.as_py() + 1 for value in values])
async def async_add_one(value: int) -> int:
return value + 1
async def async_pandas(values: pd.Series) -> pd.Series:
return values + 1
class PandasCallable:
def __call__(self, values: pd.Series) -> pd.Series:
return values + 1
class PandasScalarFunction(ScalarFunction):
def eval(self, *values: pd.Series) -> pd.Series:
value, = values
return value + 1
class AsyncScalarClass(AsyncScalarFunction):
async def eval(self, *values: int) -> int:
value, = values
return value + 1
class WrappedPandasContext:
@method_decorator
def __call__(self, context: pd.Series, value: int) -> int:
return value
wrapped_pandas_context = WrappedPandasContext()
declarations = [
(
"inferred pandas",
lambda: pf.udf(pandas_add_one, return_dtype=pf.DataType.int64()),
"pandas",
False,
),
(
"bound pandas annotation is ignored",
lambda: pf.udf(
functools.partial(with_pandas_context, pd.Series([1])),
),
"general",
False,
),
(
"bound wrapped pandas annotation is ignored",
lambda: pf.udf(
functools.partial(
wrapped_pandas_context.__call__, pd.Series([1])
),
),
"general",
False,
),
(
"pandas forward reference",
lambda: pf.udf(
pandas_forward_reference,
return_dtype=pf.DataType.int64(),
),
"pandas",
False,
),
(
"unresolved annotation does not hide pandas annotation",
lambda: pf.udf(
pandas_with_unresolved_annotation,
return_dtype=pf.DataType.int64(),
),
"pandas",
False,
),
(
"missing annotation attribute does not hide pandas annotation",
lambda: pf.udf(
pandas_after_missing_attribute,
return_dtype=pf.DataType.int64(),
),
"pandas",
False,
),
(
"missing annotation attribute falls back to general",
lambda: pf.udf(
only_missing_attribute,
return_dtype=pf.DataType.int64(),
),
"general",
False,
),
(
"any pandas annotation selects pandas",
lambda: pf.udf(mixed, return_dtype=pf.DataType.int64()),
"pandas",
False,
),
(
"explicit general wins",
lambda: pf.udf(
pandas_add_one,
return_dtype=pf.DataType.int64(),
func_type="general",
),
"general",
False,
),
(
"inferred arrow",
lambda: pf.udf(arrow_add_one, return_dtype=pf.DataType.int64()),
"arrow",
False,
),
(
"async general",
lambda: pf.udf(async_add_one),
"general",
True,
),
(
"pandas callable class",
lambda: pf.udf(
PandasCallable,
return_dtype=pf.DataType.int64(),
),
"pandas",
False,
),
(
"pandas scalar-function class",
lambda: pf.udf(
PandasScalarFunction,
return_dtype=pf.DataType.int64(),
),
"pandas",
False,
),
(
"async scalar-function class",
lambda: pf.udf(AsyncScalarClass),
"general",
True,
),
]
for case_name, declare, expected_type, expected_async in declarations:
with self.subTest(case=case_name):
wrapped = declare()
self.assertEqual(wrapped._func_type, expected_type)
self.assertEqual(
wrapped._runtime_source.is_async, expected_async
)
invalid_declarations = [
(
"async inferred pandas",
lambda: pf.udf(async_pandas, return_dtype=pf.DataType.int64()),
ValueError,
"Async scalar functions",
),
(
"async explicit pandas",
lambda: pf.udf(
async_add_one,
return_dtype=pf.DataType.int64(),
func_type="pandas",
),
ValueError,
"Async scalar functions",
),
]
for case_name, declare, error_type, message in invalid_declarations:
with self.subTest(case=case_name):
with self.assertRaisesRegex(error_type, message):
declare()
def test_sync_wrapper_around_async_target_is_rejected(self):
async def async_add_one(value: int) -> int:
return value + 1
@functools.wraps(async_add_one)
def sync_wrapper(*args, **kwargs):
return async_add_one(*args, **kwargs)
with self.assertRaisesRegex(TypeError, "async def"):
pf.udf(sync_wrapper)
def test_sync_async_scalar_eval_is_rejected(self):
class SyncAsyncScalarFunction(AsyncScalarFunction):
def eval(self, *values: int) -> int:
value, = values
return value + 1
for source in (SyncAsyncScalarFunction, SyncAsyncScalarFunction()):
with self.subTest(source=source):
with self.assertRaisesRegex(
TypeError,
"AsyncScalarFunction 'SyncAsyncScalarFunction'.*async def",
):
pf.udf(source, return_dtype=pf.DataType.int64())
def test_unrelated_methodtype_owner_requires_explicit_metadata(self):
class MethodOwner:
Batch = pd.Series
class Output(TypedDict):
value: int
def eval(self, values: "Batch") -> "Output":
return {"value": len(values)}
class ReplacedScalarFunction(ScalarFunction):
def eval(self, *values: int) -> int:
value, = values
return value
replaced = ReplacedScalarFunction()
replaced.eval = types.MethodType(MethodOwner.eval, replaced)
with self.assertRaisesRegex(
TypeError,
r"Cannot infer return_dtype for 'ReplacedScalarFunction' from its "
r"return annotation\.\nSpecify return_dtype explicitly\.",
):
pf.udf(replaced)
return_dtype = pf.DataType.struct({"value": pf.DataType.int64()})
inferred_mode = pf.udf(replaced, return_dtype=return_dtype)
self.assertEqual(inferred_mode._func_type, "general")
explicit_mode = pf.udf(
replaced,
return_dtype=return_dtype,
func_type="pandas",
)
self.assertEqual(explicit_mode._func_type, "pandas")
def test_invalid_class_invocation_descriptors_fail_eagerly(self):
class CallableBase:
def __call__(self, value: int) -> int:
return value
class HiddenCallable(CallableBase):
__call__ = None
class ScalarBase(ScalarFunction):
def eval(self, *values: int) -> int:
value, = values
return value
class HiddenScalarFunction(ScalarBase):
eval = None
class InvalidStaticCallable:
__call__ = staticmethod(None)
class InvalidClassMethodCallable:
__call__ = classmethod(None)
invalid_classes = (
(HiddenCallable, "Callable class", "__call__"),
(HiddenScalarFunction, "Scalar UDF class", "eval"),
(InvalidStaticCallable, "Callable class", "__call__"),
(InvalidClassMethodCallable, "Callable class", "__call__"),
)
for source, source_kind, method_name in invalid_classes:
with self.subTest(source=source):
message = (
rf"{source_kind} '{source.__name__}' has an unsupported "
rf"{method_name} definition\.\nDefine {method_name} as an "
r"instance, class, or static method\."
)
with self.assertRaisesRegex(TypeError, message):
pf.udf(source, return_dtype=int)
def test_descriptor_based_callable_classes_require_instances(self):
class PartialMethodCallable:
def invoke(self, offset: int, value: int) -> int:
return offset + value
__call__ = functools.partialmethod(invoke, 1)
class PartialDescriptorCallable:
__call__ = functools.partial(lambda: 1)
for source in (PartialMethodCallable, PartialDescriptorCallable):
with self.subTest(class_source=source):
with self.assertRaisesRegex(
TypeError,
rf"Callable class '{source.__name__}' has an unsupported "
r"__call__ definition\.\nDefine __call__ as an instance, "
r"class, or static method\.",
):
pf.udf(source, return_dtype=int)
with self.subTest(instance_source=source):
declaration = pf.udf(
source(), return_dtype=int, func_type="general"
)
self.assertEqual(
_return_dtype(declaration), pf.DataType.int64().not_null()
)
def test_unresolved_typed_dict_fields_have_actionable_errors(self):
class Describe:
OuterAlias = int
class Output(TypedDict):
value: Any
Output.__annotations__["value"] = "OuterAlias"
def __call__(self, value: int) -> "Output":
return {"value": value}
with self.assertRaisesRegex(
TypeError,
r"Cannot infer return_dtype for 'Describe' from its return annotation\.\n"
r"Specify return_dtype explicitly\.",
):
pf.udf(Describe)
with self.assertRaisesRegex(TypeError, "DataType or SQL"):
pf.udf(lambda value: value, return_dtype=Describe.Output)
class InvalidOutput(TypedDict):
value: Any
InvalidOutput.__annotations__["value"] = "list["
def invalid_output(value: int) -> InvalidOutput:
return {"value": value}
with self.assertRaisesRegex(
TypeError,
r"Cannot infer return_dtype for 'invalid_output' from its return "
r"annotation\.\nSpecify return_dtype explicitly\.",
):
pf.udf(invalid_output)
with self.assertRaisesRegex(TypeError, "DataType or SQL"):
pf.udf(lambda value: value, return_dtype=InvalidOutput)
def test_malformed_forward_references_have_clean_inference_behavior(self):
def pandas_after_malformed(
context: Any, values: pd.Series
) -> int:
return len(values)
pandas_after_malformed.__annotations__["context"] = "list["
self.assertEqual(
pf.udf(
pandas_after_malformed,
return_dtype=pf.DataType.int64(),
)._func_type,
"pandas",
)
def only_malformed(context: Any) -> int:
return 1
only_malformed.__annotations__["context"] = "list["
self.assertEqual(
pf.udf(
only_malformed, return_dtype=pf.DataType.int64()
)._func_type,
"general",
)
def malformed_return(value: int) -> int:
return value
malformed_return.__annotations__["return"] = "list["
with self.assertRaisesRegex(
TypeError,
r"Cannot infer return_dtype for 'malformed_return' from its return "
r"annotation\.\nSpecify return_dtype explicitly\.",
):
pf.udf(malformed_return)
def test_execution_resource_options_are_forwarded(self):
cases = [
({}, None, None),
({"concurrency": 2}, 2, None),
({"func_type": "pandas", "concurrency": 3, "batch_size": 64}, 3, 64),
({"func_type": "arrow", "batch_size": 128}, None, 128),
]
for options, concurrency, batch_size in cases:
with self.subTest(options=options):
declaration = pf.udf(
lambda value: value, return_dtype=pf.DataType.int64(), **options)
wrapper = declaration._table_udf_wrapper
self.assertEqual(wrapper._concurrency, concurrency)
self.assertEqual(wrapper._batch_size, batch_size)
def test_execution_resource_options_use_table_validation(self):
with self.assertRaisesRegex(ValueError, "concurrency must be a positive integer"):
pf.udf(lambda value: value, return_dtype=pf.DataType.int64(), concurrency=True)
def test_determinism_and_name_metadata(self):
class NonDeterministic(ScalarFunction):
def eval(self, *values: int) -> int:
value, = values
return value
def is_deterministic(self):
return False
class DefaultDeterministic(ScalarFunction):
def eval(self, *values: int) -> int:
value, = values
return value
instance = NonDeterministic()
declarations = [
(
"matching instance metadata",
lambda: pf.udf(instance, deterministic=False),
False,
),
("class default", lambda: pf.udf(DefaultDeterministic), True),
("matching class metadata", lambda: pf.udf(NonDeterministic, deterministic=False),
False),
]
for case_name, declare, expected in declarations:
with self.subTest(case=case_name):
self.assertEqual(declare()._deterministic, expected)
self.assertIs(
inspect.signature(pf.udf).parameters["deterministic"].default,
True,
)
with self.assertRaisesRegex(ValueError, "Inconsistent deterministic"):
pf.udf(instance)
named = pf.udf(instance, deterministic=False, name="identity")
self.assertEqual(named.__name__, "identity")
self.assertEqual(named._table_udf_wrapper._name, "identity")
def test_scalar_function_classes_defer_custom_determinism_validation(self):
for base in (ScalarFunction, AsyncScalarFunction):
for deterministic in (True, False):
constructed = []
class CustomDeterminism(base):
def __init__(self):
constructed.append(True)
def is_deterministic(self):
return deterministic
if base is AsyncScalarFunction:
async def eval(self, value):
return value
else:
def eval(self, value):
return value
class InheritedDeterminism(CustomDeterminism):
pass
for source in (CustomDeterminism, InheritedDeterminism):
with self.subTest(base=base, deterministic=deterministic, source=source):
pf.udf(source, return_dtype=int, deterministic=deterministic)
self.assertEqual(constructed, [])
declaration = pf.udf(
CustomDeterminism(), return_dtype=int, deterministic=deterministic)
self.assertEqual(declaration._deterministic, deterministic)
def test_general_structured_results_are_normalized_recursively(self):
from pyflink.dataframe.udf import _create_result_normalizer
class Details:
__slots__ = ("label", "scores")
def __init__(self, label, scores):
self.label = label
self.scores = scores
class ItemsOnly:
def __init__(self, items):
self._items = items
def items(self):
return self._items
class PropertyDetails:
def __init__(self, label, scores):
self._label = label
self.scores = scores
@property
def label(self):
return self._label.upper()
class MissingLabelDetails:
def __init__(self, scores):
self.scores = scores
class FailingPropertyDetails:
scores = [17]
@property
def label(self):
raise AttributeError("label lookup failed")
@dataclass
class Result:
id: int
details: Details
attributes: dict
return_dtype = pf.DataType.struct(
{
"id": pf.DataType.int64(),
"details": pf.DataType.struct(
{
"label": pf.DataType.string(),
"scores": pf.DataType.list(pf.DataType.int64()),
}
),
"attributes": pf.DataType.map(
pf.DataType.string(), pf.DataType.int64()
),
}
)
table_type = return_dtype._to_table_data_type()
self.assertIsInstance(table_type, RowType)
result_normalizer = _create_result_normalizer(table_type)
self.assertIsNotNone(result_normalizer)
named_row = Row(
id=4,
details=Row(label="named", scores=[5]),
attributes={"count": 6},
)
named_row.set_row_kind(RowKind.DELETE)
expected_named_row = Row(
id=4,
details=Row(label="named", scores=[5]),
attributes={"count": 6},
)
expected_named_row.set_row_kind(RowKind.DELETE)
cases = [
(
"mapping",
{
"id": 1,
"details": {"scores": (2, 3), "ignored": "extra"},
"attributes": [("answer", 42)],
"ignored": "extra",
},
Row(
id=1,
details=Row(label=None, scores=[2, 3]),
attributes={"answer": 42},
),
),
(
"named row",
named_row,
expected_named_row,
),
(
"positional list and tuple",
[7, ("positional", (8, 9)), {"count": 10}],
Row(
id=7,
details=Row(label="positional", scores=[8, 9]),
attributes={"count": 10},
),
),
(
"dataclass and attribute objects",
Result(
id=11,
details=Details(label="object", scores=[12]),
attributes=ItemsOnly([("count", 13)]),
),
Row(
id=11,
details=Row(label="object", scores=[12]),
attributes={"count": 13},
),
),
(
"property attribute",
Result(
id=14,
details=PropertyDetails(label="property", scores=[15]),
attributes={"count": 16},
),
Row(
id=14,
details=Row(label="PROPERTY", scores=[15]),
attributes={"count": 16},
),
),
(
"missing object attribute",
Result(
id=18,
details=MissingLabelDetails(scores=[19]),
attributes={"count": 20},
),
Row(
id=18,
details=Row(label=None, scores=[19]),
attributes={"count": 20},
),
),
]
for case_name, value, expected in cases:
with self.subTest(case=case_name):
self.assertEqual(
result_normalizer(value), expected
)
with self.assertRaisesRegex(ValueError, "Expected 3 value"):
result_normalizer((1, 2))
with self.assertRaisesRegex(TypeError, "Expected a Mapping"):
result_normalizer(object())
with self.assertRaisesRegex(AttributeError, "label lookup failed"):
result_normalizer(
{
"id": 21,
"details": FailingPropertyDetails(),
"attributes": {},
},
)
def test_invalid_declarations_fail_eagerly(self):
def missing_return(value):
return value
def unresolved_return(value):
return value
unresolved_return.__annotations__ = {
"return": "UnavailableReturn"
}
def pandas_identity(values: pd.Series) -> pd.Series:
return values
class RequiresArgument:
def __init__(self, value):
self.value = value
def __call__(self, other: int) -> int:
return other + self.value
class RequiresScalarArgument(ScalarFunction):
def __init__(self, value):
self.value = value
def eval(self, *values: int) -> int:
value, = values
return value + self.value
class RequiresAsyncScalarArgument(AsyncScalarFunction):
def __init__(self, value):
self.value = value
async def eval(self, *values: int) -> int:
value, = values
return value + self.value
class NotCallable:
pass
class NonScalarFunction(TableFunction):
def eval(self, value):
return value
class MissingCallableReturn:
def __call__(self, value):
return value
class MissingScalarReturn(ScalarFunction):
def eval(self, value):
return value
invalid_declarations = [
(
"not callable",
lambda: pf.udf(42, return_dtype=pf.DataType.int64()),
TypeError,
"func must be callable",
),
(
"non-callable class",
lambda: pf.udf(NotCallable, return_dtype=pf.DataType.int64()),
TypeError,
"func must be callable",
),
(
"non-scalar UDF class",
lambda: pf.udf(
NonScalarFunction,
return_dtype=pf.DataType.int64(),
),
TypeError,
"func must be a scalar UDF",
),
(
"missing return",
lambda: pf.udf(missing_return),
TypeError,
"add a return annotation",
),
(
"unresolved return",
lambda: pf.udf(unresolved_return),
TypeError,
r"from its return annotation\.\nSpecify return_dtype explicitly\.",
),
(
"callable class missing return",
lambda: pf.udf(MissingCallableReturn),
TypeError,
"add a return annotation",
),
(
"scalar function class missing return",
lambda: pf.udf(MissingScalarReturn),
TypeError,
"add a return annotation",
),
(
"Table return type",
lambda: pf.udf(
missing_return, return_dtype=TableDataTypes.BIGINT()
),
TypeError,
"return_dtype must be",
),
(
"required constructor argument",
lambda: pf.udf(RequiresArgument),
TypeError,
"zero-argument constructor",
),
(
"required scalar constructor argument",
lambda: pf.udf(RequiresScalarArgument),
TypeError,
"zero-argument constructor",
),
(
"required async scalar constructor argument",
lambda: pf.udf(RequiresAsyncScalarArgument),
TypeError,
"zero-argument constructor",
),
(
"invalid determinism",
lambda: pf.udf(
missing_return,
return_dtype=pf.DataType.int64(),
deterministic=1,
),
TypeError,
"deterministic must be",
),
(
"invalid name",
lambda: pf.udf(
missing_return,
return_dtype=pf.DataType.int64(),
name=1,
),
TypeError,
"name must be",
),
(
"empty name",
lambda: pf.udf(
missing_return,
return_dtype=pf.DataType.int64(),
name="",
),
ValueError,
"name must not be empty",
),
(
"unsupported func type",
lambda: pf.udf(
missing_return,
return_dtype=pf.DataType.int64(),
func_type="unsupported",
),
ValueError,
"func_type must be one of",
),
(
"pandas return type required",
lambda: pf.udf(pandas_identity),
TypeError,
"return_dtype is required",
),
]
for case_name, declare, error_type, message in invalid_declarations:
with self.subTest(case=case_name):
with self.assertRaisesRegex(error_type, message):
declare()
class DataFrameUDFAdapterTests(unittest.TestCase):
def pandas_map_output(self, adapter, dtype, *inputs):
from pyflink.table.utils import create_pandas_table_row_schema, pandas_table_row_to_arrow
import pytz
row_type = dtype._to_table_data_type()
schema = create_pandas_table_row_schema(TableDataTypes.ROW([
TableDataTypes.FIELD("result", row_type)]))
return pandas_table_row_to_arrow(
schema, pytz.UTC, row_type, adapter.eval(*inputs)).column(0).to_pylist()
def test_pandas_map_batches_returns_flattened_dataframe(self):
from pyflink.dataframe.udf import _resolve_map_udf
dtype = pf.DataType.struct({
"v": pf.DataType.int64(),
"payload": pf.DataType.struct({"v": pf.DataType.int64(),
"tail": pf.DataType.int64()}),
"other": pf.DataType.struct({"v": pf.DataType.string()}),
"tail": pf.DataType.int64(),
})
input_type = TableDataTypes.ROW([
TableDataTypes.FIELD("tag", TableDataTypes.STRING()),
TableDataTypes.FIELD("id", TableDataTypes.BIGINT())])
def transform(batch):
self.assertEqual(batch["tag"].tolist(), ["x"] * len(batch["id"]))
return {"tail": batch["id"] + 10,
"other": pd.DataFrame({"v": ["a", None]}, index=batch["id"].index),
"payload": pd.DataFrame({
"v": pd.Series([9007199254740993, None],
index=batch["id"].index, dtype=object),
"tail": batch["id"] + 20}),
"v": batch["id"].sort_index(ascending=False)}
adapter = _resolve_map_udf(transform, dtype, "pandas", input_type)._func
adapter.open(object())
try:
result = adapter.eval("x", pd.Series([1, 2]))
self.assertIsInstance(result, pd.DataFrame)
pd.testing.assert_frame_equal(result, pd.concat([
pd.Series([1, 2], name="v"),
pd.Series([9007199254740993, None], name="v", dtype=object),
pd.Series([21, 22], name="tail"),
pd.Series(["a", None], name="v"),
pd.Series([11, 12], name="tail"),
], axis=1))
self.assertEqual(self.pandas_map_output(adapter, dtype, "x", pd.Series([1, 2])), [
{"v": 1, "payload": {"v": 9007199254740993, "tail": 21},
"other": {"v": "a"}, "tail": 11},
{"v": 2, "payload": {"v": None, "tail": 22},
"other": {"v": None}, "tail": 12},
])
finally:
adapter.close()
def test_map_bindings_leave_other_uses_of_a_declaration_unchanged(self):
from pyflink.dataframe.udf import _resolve_map_udf
@pf.udf(return_dtype=pf.DataType.struct({"total": pf.DataType.int64()}))
def total(value):
return {"total": sum(value.values()) if isinstance(value, dict) else value}
expression = total._table_udf_wrapper._func
maps = [_resolve_map_udf(total, None, "general", TableDataTypes.ROW([
TableDataTypes.FIELD(name, TableDataTypes.BIGINT()) for name in names
]))._func for names in (["left", "right"], ["value"])]
for adapter in [expression] + maps:
adapter.open(object())
self.addCleanup(adapter.close)
self.assertEqual(maps[0].eval(1, 2), Row(total=3))
self.assertEqual(maps[1].eval(4), Row(total=4))
self.assertEqual(maps[0].eval(5, 6), Row(total=11))
self.assertEqual(expression.eval(7), Row(total=7))
def test_map_requires_a_dictionary_result(self):
from pyflink.dataframe.udf import _resolve_map_udf
dtype = pf.DataType.struct({"value": pf.DataType.int64()})
for result in (None, (1,), 1):
with self.subTest(result=result):
adapter = _resolve_map_udf(
lambda row: result, dtype, "general", dtype._to_table_data_type())._func
adapter.open(object())
try:
with self.assertRaisesRegex(TypeError, "dictionary"):
adapter.eval(1)
finally:
adapter.close()
def test_pandas_map_batches_preserves_row_columns_for_transported_and_fused_inputs(self):
from pyflink.dataframe.udf import _resolve_map_udf
from pyflink.table.types import create_arrow_schema
from pyflink.table.utils import arrow_to_pandas
import pytz
input_type = TableDataTypes.ROW([
TableDataTypes.FIELD("id", TableDataTypes.BIGINT()),
TableDataTypes.FIELD("payload", TableDataTypes.ROW([
TableDataTypes.FIELD("left", TableDataTypes.BIGINT()),
TableDataTypes.FIELD("right", TableDataTypes.BIGINT())])),
TableDataTypes.FIELD("tail", TableDataTypes.BIGINT()),
])
dtype = pf.DataType.struct({"score": pf.DataType.int64()})
def transform(batch):
self.assertEqual(list(batch), ["id", "payload", "tail"])
payload = batch["payload"]
self.assertIsInstance(payload, pd.DataFrame)
self.assertEqual(list(payload), ["left", "right"])
pd.testing.assert_frame_equal(payload, pd.DataFrame(
{"left": [2, None, 4], "right": [3, None, None]},
index=payload.index))
return {"score": batch["tail"] + batch["id"]}
adapter = _resolve_map_udf(transform, dtype, "pandas", input_type)._func
adapter.open(object())
try:
grouped = pd.DataFrame({
"id": [1, 2, 3], "tail": [10, 20, 30], "payload": [
{"left": 2, "right": 3}, None,
{"left": 4, "right": None},
],
})[["id", "payload", "tail"]]
schema = create_arrow_schema(input_type.field_names(), input_type.field_types())
batch = pa.RecordBatch.from_pandas(grouped, schema=schema)
transported = arrow_to_pandas(pytz.UTC, input_type.field_types(), [batch])
self.assertEqual(self.pandas_map_output(adapter, dtype, *transported),
[{"score": 11}, {"score": 22}, {"score": 33}])
for names in (["unrelated1", "unrelated2"], ["right", "left"]):
fused_payload = transported[1].set_axis(names, axis=1)
self.assertEqual(self.pandas_map_output(
adapter, dtype, transported[0], fused_payload, transported[2]),
[{"score": 11}, {"score": 22}, {"score": 33}])
self.assertEqual(list(fused_payload), names)
grouped.index = [2, 0, 1]
with self.assertRaisesRegex(TypeError, "ROW.*pandas.DataFrame"):
adapter.eval(*(grouped[name] for name in input_type.field_names()))
fused = [column.set_axis(grouped.index, axis=0) for column in transported]
self.assertEqual(self.pandas_map_output(adapter, dtype, *fused),
[{"score": 11}, {"score": 22}, {"score": 33}])
finally:
adapter.close()
def test_pandas_map_batches_preserves_nested_collections_with_repeated_names(self):
import cloudpickle
from pyflink.dataframe.udf import _resolve_map_udf
from pyflink.fn_execution import flink_fn_execution_pb2 as proto
from pyflink.fn_execution.table.operations import ScalarFunctionOperation
from pyflink.table.udf import DelegatingScalarFunction
from pyflink.table.utils import arrow_to_pandas
import pytz
row_type = TableDataTypes.ROW([TableDataTypes.FIELD(
"payload", TableDataTypes.MAP(TableDataTypes.STRING(), TableDataTypes.BIGINT()))])
input_type = TableDataTypes.ROW([TableDataTypes.FIELD("payload", row_type)])
dtype = pf.DataType.struct({"size": pf.DataType.int64()})
def transform(batch):
self.assertEqual(batch["payload"]["payload"].tolist(), [[("x", 1), ("y", 2)], []])
return {"size": batch["payload"]["payload"].map(len)}
batch = pa.record_batch([pa.array([
{"payload": [("x", 1), ("y", 2)]}, {"payload": []},
], type=pa.struct([("payload", pa.map_(pa.string(), pa.int64()))]))], names=["payload"])
columns = arrow_to_pandas(pytz.UTC, [row_type], [batch])
adapter = _resolve_map_udf(transform, dtype, "pandas", input_type)._func
adapter.open(object())
try:
self.assertEqual(self.pandas_map_output(adapter, dtype, columns[0]),
[{"size": 2}, {"size": 0}])
nested_dtype = pf.DataType.struct({"payload": pf.DataType.struct({
"payload": pf.DataType.map(pf.DataType.string(), pf.DataType.int64())})})
producer = _resolve_map_udf(
lambda batch: {"payload": batch["payload"]},
nested_dtype, "pandas", input_type)._func
producer_proto = proto.UserDefinedFunction(
payload=cloudpickle.dumps(producer), is_pandas_udf=True,
inputs=[proto.Input(inputOffset=0)])
producer.open(object())
try:
self.assertEqual(self.pandas_map_output(producer, nested_dtype, columns[0]),
[{"payload": {"payload": [("x", 1), ("y", 2)]}},
{"payload": {"payload": []}}])
finally:
producer.close()
for values, expected in ((columns, [2, 0]),
([column.head(0) for column in columns], [])):
for shared in (False, True):
with self.subTest(empty_batch=not expected, shared=shared):
consumer = proto.UserDefinedFunction(
payload=cloudpickle.dumps(DelegatingScalarFunction(
lambda frame: frame["payload"].map(len))),
is_pandas_udf=True, takes_row_as_input=True,
inputs=[proto.Input(refIndex=0)] if shared
else [proto.Input(udf=producer_proto)])
operation = ScalarFunctionOperation(proto.UserDefinedFunctions(
udfs=[producer_proto, consumer] if shared else [consumer],
output_indices=[1] if shared else []))
operation.open()
try:
self.assertEqual(operation.process_element(values)[0].tolist(),
expected)
finally:
operation.close()
finally:
adapter.close()
def test_pandas_map_batches_preserves_map_inputs(self):
from pyflink.dataframe.udf import _resolve_map_udf
from pyflink.table.types import create_arrow_schema
from pyflink.table.utils import arrow_to_pandas, pandas_to_arrow
import pytz
lookup = TableDataTypes.MAP(TableDataTypes.STRING(), TableDataTypes.BIGINT())
payload = TableDataTypes.ROW([
TableDataTypes.FIELD("lookup", lookup),
])
input_type = TableDataTypes.ROW([
TableDataTypes.FIELD("lookup", lookup), TableDataTypes.FIELD("payload", payload)])
dtype = pf.DataType.struct({"total": pf.DataType.int64()})
def transform(batch):
self.assertIs(batch["lookup"], inputs[0])
self.assertIs(batch["payload"], inputs[1])
return {"total": batch["lookup"].map(
lambda value: 0 if value is None else sum(value.values())
if isinstance(value, dict) else sum(item for _, item in value))}
columns = [pd.Series([{"top": 1}, {}, None], dtype=object), pd.DataFrame({
"lookup": [{"field": 2}, [("same", 3), ("same", 4)], None],
})]
schema = create_arrow_schema(input_type.field_names(), input_type.field_types())
transported = arrow_to_pandas(pytz.UTC, input_type.field_types(), [pandas_to_arrow(
schema, pytz.UTC, input_type.field_types(), columns)])
adapter = _resolve_map_udf(transform, dtype, "pandas", input_type)._func
adapter.open(object())
try:
for inputs in (columns, transported):
self.assertEqual(self.pandas_map_output(adapter, dtype, *inputs),
[{"total": 1}, {"total": 0}, {"total": 0}])
finally:
adapter.close()
def test_arrow_map_batches_preserves_nested_and_chunked_columns(self):
from pyflink.dataframe.udf import _resolve_map_udf
from pyflink.fn_execution.utils.operation_utils import check_arrow_udf_result
dtype = pf.DataType.struct({
"id": pf.DataType.int64(),
"payload": pf.DataType.struct({"x": pf.DataType.int64()}),
})
batch = pa.StructArray.from_arrays([
pa.array([1, 2]), pa.array([{"x": None}, None], type=pa.struct([("x", pa.int64())])),
pa.array([{"inner": {"v": 1}}, {"inner": {"v": 2}}],
type=pa.struct([("inner", pa.struct([("v", pa.int64())]))])),
pa.array([[{"v": 1}], []], type=pa.list_(pa.struct([("v", pa.int64())])))
], names=["id", "payload", "deep", "items"])
def transform(columns):
self.assertEqual(columns["tag"].to_pylist(), ["x"] * len(columns["id"]))
self.assertIsInstance(columns["payload"], pa.StructArray)
self.assertEqual(columns["deep"].field("inner").field("v").to_pylist(),
[1, 2][:len(columns["id"])])
self.assertEqual(columns["items"].to_pylist(),
[[{"v": 1}], []][:len(columns["id"])])
return {"payload": columns["payload"], "ignored": pa.array([0, 0]),
"id": pa.chunked_array([columns["id"].slice(0, 1),
columns["id"].slice(1)])}
input_type = TableDataTypes.ROW([
TableDataTypes.FIELD("tag", TableDataTypes.STRING())
] + dtype._to_table_data_type().fields + [
TableDataTypes.FIELD("deep", TableDataTypes.ROW([
TableDataTypes.FIELD("inner", TableDataTypes.ROW([
TableDataTypes.FIELD("v", TableDataTypes.BIGINT())]))])),
TableDataTypes.FIELD("items", TableDataTypes.ARRAY(TableDataTypes.ROW([
TableDataTypes.FIELD("v", TableDataTypes.BIGINT())]))),
])
adapter = _resolve_map_udf(transform, dtype, "arrow", input_type)._func
adapter.open(object())
try:
result = check_arrow_udf_result(adapter.eval, "x", *batch.flatten())
self.assertEqual(result.to_pylist(),
[{"id": 1, "payload": {"x": None}}, {"id": 2, "payload": None}])
self.assertEqual(adapter.eval("x", *batch.slice(0, 0).flatten()).to_pylist(), [])
finally:
adapter.close()
def test_pandas_map_batches_aligns_indexes_and_checks_cardinality(self):
from pyflink.dataframe.udf import _resolve_map_udf
dtype = pf.DataType.struct({"score": pf.DataType.int64(), "id": pf.DataType.int64()})
def align(batch):
self.assertIsInstance(batch, dict)
self.assertIsInstance(batch["id"], pd.Series)
return {"score": pd.Series([100, 200], index=batch["id"].index[::-1]).rename_axis(None),
"id": batch["id"],
"ignored": pd.Series([0, 0])}
def wrong_length(batch):
return {"id": batch["id"], "score": pd.Series([100, 200], index=[1, 2])}
def too_short(batch):
return {"id": batch["id"].head(1), "score": batch["id"].head(1)}
def same_order(batch):
return {"score": pd.Series([200, 100], index=batch["id"].index).rename_axis(None),
"id": batch["id"]}
def partial(batch):
return {"score": pd.Series([9007199254740993], index=[0]), "id": batch["id"]}
input_type = TableDataTypes.ROW([TableDataTypes.FIELD("id", TableDataTypes.BIGINT())])
for source, index in ((align, pd.RangeIndex(2)), (align, pd.Index([2, 0], name="rows")),
(align, pd.to_datetime(["2024-01-03", "2024-01-01"])),
(same_order, pd.Index([2, 0], name="rows")),
(partial, pd.RangeIndex(2)),
(wrong_length, pd.RangeIndex(2)), (too_short, pd.RangeIndex(2)),
(functools.partial(too_short), pd.RangeIndex(2))):
adapter = _resolve_map_udf(source, dtype, "pandas", input_type)._func
adapter.open(object())
try:
batch = pd.DataFrame({"id": [10, 20]}, index=index)
if source in (wrong_length, too_short, partial) \
or isinstance(source, functools.partial):
with self.assertRaisesRegex(ValueError, "length"):
adapter.eval(batch["id"])
else:
self.assertEqual(self.pandas_map_output(adapter, dtype, batch["id"]),
[{"score": 200, "id": 10}, {"score": 100, "id": 20}])
finally:
adapter.close()
def test_pandas_map_batches_requires_row_fields_including_collections(self):
from pyflink.dataframe.udf import _resolve_map_udf
from pyflink.table.utils import create_pandas_table_row_schema, pandas_table_row_to_arrow
import pytz
details = pf.DataType.struct({"name": pf.DataType.string()})
dtype = pf.DataType.struct({
"details": details,
"items": pf.DataType.list(
pf.DataType.map(pf.DataType.string(), details)),
"plain_items": pf.DataType.list(
pf.DataType.map(pf.DataType.string(), pf.DataType.int64())),
"lookup": pf.DataType.map(pf.DataType.string(), details),
})
input_type = TableDataTypes.ROW([TableDataTypes.FIELD("id", TableDataTypes.BIGINT())])
valid = {"details": pd.DataFrame({"name": [None]}),
"items": [None, {"item key": {"name": "a"}}],
"plain_items": [None, {"plain key": 1}, {}],
"lookup": {"any key": {"name": None}}, "extra": 2}
cases = [(valid, None), ({"items": [], "lookup": {}}, KeyError),
(dict(valid, details=pd.DataFrame(index=[0])), ValueError),
(dict(valid, details=pd.DataFrame({"name": [None], "extra": [1]})), ValueError),
(dict(valid, items=[{"item key": {}}]), KeyError),
(dict(valid, lookup={"any key": {}}), KeyError),
(dict(valid, lookup={"any key": types.SimpleNamespace()}), AttributeError),
(dict(valid, lookup={"any key": ()}), ValueError),
(dict(valid, lookup={"any key": (None, "extra")}), ValueError)]
for result, error in cases:
with self.subTest(result=result):
adapter = _resolve_map_udf(
lambda batch: {name: value if isinstance(value, pd.DataFrame)
else pd.Series([value]) for name, value in result.items()},
dtype, "pandas", input_type)._func
adapter.open(object())
try:
if result is valid:
output = adapter.eval(pd.Series([1]))
self.assertEqual(output["items"].iloc[0],
[None, [("item key", {"name": "a"})]])
self.assertEqual(output["plain_items"].iloc[0],
[None, [("plain key", 1)], []])
self.assertEqual(valid["items"], [None, {"item key": {"name": "a"}}])
self.assertEqual(valid["plain_items"], [None, {"plain key": 1}, {}])
row_type = dtype._to_table_data_type()
schema = create_pandas_table_row_schema(TableDataTypes.ROW([
TableDataTypes.FIELD("result", row_type)]))
batch = pandas_table_row_to_arrow(schema, pytz.UTC, row_type, output)
self.assertEqual(batch.to_pylist(), [{"result": {
"details": {"name": None},
"items": [None, [("item key", {"name": "a"})]],
"plain_items": [None, [("plain key", 1)], []],
"lookup": [("any key", {"name": None})]}}])
else:
with self.assertRaises(error):
adapter.eval(pd.Series([1]))
finally:
adapter.close()
def test_pandas_map_batches_preserves_nullable_structured_columns(self):
import tracemalloc
from pyflink.dataframe.udf import _create_pandas_result_normalizer, _resolve_map_udf
row_type = TableDataTypes.ROW([TableDataTypes.FIELD("v", TableDataTypes.BIGINT())])
for data_type, payload in (
(TableDataTypes.ARRAY(row_type), [{"v": i} for i in range(10000)]),
(TableDataTypes.MAP(TableDataTypes.STRING(), row_type),
[(str(i), {"v": i}) for i in range(10000)]),
):
normalize = _create_pandas_result_normalizer(data_type)
normalize(payload)
tracemalloc.start()
try:
normalized = normalize(payload)
peak = tracemalloc.get_traced_memory()[1]
finally:
tracemalloc.stop()
self.assertIs(normalized, payload)
self.assertLess(peak, 16384)
original = Row(v=1)
if isinstance(data_type, MapType):
source = [("same", payload[0][1]), ("changed", original)]
converted = normalize(source)
self.assertIs(converted[0][1], source[0][1])
self.assertEqual(converted[1], ("changed", {"v": 1}))
self.assertIs(source[1][1], original)
else:
source = [payload[0], original]
converted = normalize(source)
self.assertIs(converted[0], source[0])
self.assertEqual(converted[1], {"v": 1})
self.assertIs(source[1], original)
dtype = pf.DataType.struct({
"details": pf.DataType.struct({"name": pf.DataType.string()}),
"items": pf.DataType.list(pf.DataType.int64()),
"lookup": pf.DataType.map(pf.DataType.string(), pf.DataType.int64()),
})
input_type = TableDataTypes.ROW([TableDataTypes.FIELD("id", TableDataTypes.BIGINT())])
class RowFrame(pd.DataFrame):
pass
for frame_type in (pd.DataFrame, RowFrame):
with self.subTest(frame_type=frame_type):
adapter = _resolve_map_udf(
lambda batch: {"details": frame_type({"name": [None, float("nan"), pd.NA]}),
"items": pd.Series([None, float("nan"), pd.NA]),
"lookup": pd.Series([None, float("nan"), pd.NA])},
dtype, "pandas", input_type)._func
adapter.open(object())
try:
self.assertEqual(self.pandas_map_output(adapter, dtype, pd.Series([1, 2, 3])),
[{"details": {"name": None}, "items": None, "lookup": None}]
* 3)
finally:
adapter.close()
def test_pandas_map_batches_requires_matching_output_containers(self):
from pyflink.dataframe.udf import _resolve_map_udf
input_type = TableDataTypes.ROW([TableDataTypes.FIELD("id", TableDataTypes.BIGINT())])
for field_type, columns, container in (
(pf.DataType.struct({"name": pf.DataType.string()}),
(pd.Series([{"name": "a"}]), None, [1]), "DataFrame"),
(pf.DataType.int64(), (pd.DataFrame({"inner": [1]}), None, [1]), "Series"),
):
dtype = pf.DataType.struct({"payload": field_type})
for column in columns:
with self.subTest(column=column, field_type=field_type):
adapter = _resolve_map_udf(
lambda batch: {"payload": column}, dtype, "pandas", input_type)._func
adapter.open(object())
try:
with self.assertRaisesRegex(TypeError, f"pandas.{container}"):
adapter.eval(pd.Series([1]))
finally:
adapter.close()
def test_pandas_map_batches_rejects_one_shot_structured_collections(self):
from pyflink.dataframe.udf import _resolve_map_udf
details = pf.DataType.struct({"name": pf.DataType.string()})
input_type = TableDataTypes.ROW([TableDataTypes.FIELD("id", TableDataTypes.BIGINT())])
for field_type, value in (
(pf.DataType.list(pf.DataType.map(pf.DataType.string(), details)),
iter([{"key": {"name": "a"}}])),
(pf.DataType.map(pf.DataType.string(), details), iter([("key", {"name": "a"})])),
):
with self.subTest(field_type=field_type):
dtype = pf.DataType.struct({"value": field_type})
adapter = _resolve_map_udf(
lambda batch: {"value": pd.Series([value])}, dtype, "pandas", input_type)._func
adapter.open(object())
try:
with self.assertRaisesRegex(TypeError, "iterator"):
adapter.eval(pd.Series([1]))
finally:
adapter.close()
def test_map_requires_output_fields_even_when_nullable(self):
from pyflink.dataframe.udf import _resolve_map_udf
class Output(TypedDict):
value: Optional[str]
def transform(row) -> Output:
return {"value": None} if row["id"] else {}
input_type = TableDataTypes.ROW([TableDataTypes.FIELD("id", TableDataTypes.BIGINT())])
explicit = pf.DataType.struct({"value": pf.DataType.string()})
for source, dtype in (
(transform, None), (transform, Output),
(pf.udf(transform), None), (transform, explicit),
):
with self.subTest(source=source, dtype=dtype):
adapter = _resolve_map_udf(source, dtype, "general", input_type)._func
adapter.open(object())
try:
self.assertEqual(adapter.eval(1), Row(value=None))
with self.assertRaisesRegex(KeyError, "value"):
adapter.eval(0)
finally:
adapter.close()
def test_map_requires_nested_row_fields_including_collections(self):
from pyflink.dataframe.udf import _resolve_map_udf
details = pf.DataType.struct({"name": pf.DataType.string()})
dtype = pf.DataType.struct({
"details": details,
"items": pf.DataType.list(details),
"lookup": pf.DataType.map(pf.DataType.string(), details),
})
input_type = TableDataTypes.ROW([TableDataTypes.FIELD("id", TableDataTypes.BIGINT())])
valid = {"details": {"name": None, "extra": 1}, "items": [None, {"name": "a"}],
"lookup": {"any key": {"name": None}}, "extra": 2}
for result in (valid, dict(valid, details={}), dict(valid, items=[{}]),
dict(valid, lookup={"any key": {}}),
dict(valid, details=types.SimpleNamespace())):
def transform(row):
return result
async def transform_async(row):
return result
for source in (transform, transform_async):
with self.subTest(result=result, source=source):
adapter = _resolve_map_udf(source, dtype, "general", input_type)._func
adapter.open(object())
try:
def invoke():
value = adapter.eval(1)
return asyncio.run(value) if inspect.isawaitable(value) else value
if result is valid:
self.assertEqual(invoke(), Row(
details=Row(name=None), items=[None, Row(name="a")],
lookup={"any key": Row(name=None)}))
else:
with self.assertRaisesRegex((KeyError, AttributeError), "name"):
invoke()
finally:
adapter.close()
def test_row_maps_receive_named_fields_and_project_output_schema(self):
from pyflink.dataframe.udf import (
_DataFrameAsyncScalarFunctionAdapter,
_DataFrameScalarFunctionAdapter,
_UDFUsage,
_resolve_udf,
)
def transform(row):
self.assertEqual(row, {"value": 3, "label": "a"})
return {"label": row["label"].upper(), "extra": 99, "value": row["value"] + 1}
async def transform_async(row):
return transform(row)
dtype = pf.DataType.struct({"value": pf.DataType.int64(), "label": pf.DataType.string()})
for source, adapter_type in (
(transform, _DataFrameScalarFunctionAdapter),
(transform_async, _DataFrameAsyncScalarFunctionAdapter),
):
with self.subTest(source=source):
adapter = adapter_type(
_resolve_udf(source).runtime_source, dtype, True, _UDFUsage.MAP, "general",
input_type=dtype._to_table_data_type(),
)
adapter.open(object())
try:
result = adapter.eval(3, "a")
if inspect.isawaitable(result):
result = asyncio.run(result)
self.assertEqual(result, Row(value=4, label="A"))
finally:
adapter.close()
def test_row_maps_preserve_resolved_column_inputs(self):
import cloudpickle
from pyflink.dataframe.udf import _resolve_map_udf
from pyflink.fn_execution import flink_fn_execution_pb2 as proto
from pyflink.fn_execution.table.operations import ScalarFunctionOperation
from pyflink.serializers import PickleSerializer
from pyflink.table.udf import DelegatingScalarFunction
literal = proto.Input(inputConstant=b"\x00" + PickleSerializer().dumps("xyz"))
increment = proto.UserDefinedFunction(
payload=cloudpickle.dumps(DelegatingScalarFunction(lambda value: value + 1)),
inputs=[proto.Input(inputOffset=0)])
for fields, inputs, values, expected in (
({"tag": pf.DataType.string()}, [literal], [], Row(tag="xyz")),
({"id": pf.DataType.int64(), "tag": pf.DataType.string()},
[proto.Input(inputOffset=0), literal], [1], Row(id=1, tag="xyz")),
({"first": pf.DataType.int64(), "second": pf.DataType.int64()},
[proto.Input(udf=increment), proto.Input(udf=increment)], [1],
Row(first=2, second=2)),
({"payload": pf.DataType.struct({"tag": pf.DataType.string()})},
[proto.Input(inputOffset=0)], [Row(tag="xyz")], Row(payload=Row(tag="xyz"))),
):
with self.subTest(fields=fields):
dtype = pf.DataType.struct(fields)
function = _resolve_map_udf(
lambda row: row, dtype, "general", dtype._to_table_data_type())
operation = ScalarFunctionOperation(proto.UserDefinedFunctions(udfs=[
proto.UserDefinedFunction(
payload=cloudpickle.dumps(function._func), inputs=inputs,
takes_row_as_input=function._takes_row_as_input)]))
operation.open()
try:
self.assertEqual(operation.process_element(values), [expected])
finally:
operation.close()
def test_general_result_normalizers_are_bound_by_return_type(self):
from pyflink.dataframe.udf import (
_DataFrameAsyncScalarFunctionAdapter,
_DataFrameScalarFunctionAdapter,
_UDFUsage,
_resolve_udf,
)
return_dtype = pf.DataType.struct(
{
"value": pf.DataType.int64(),
"labels": pf.DataType.list(pf.DataType.string()),
}
)
def describe(value):
return {"value": value, "labels": (str(value),)}
async def describe_async(value):
return {"value": value, "labels": (str(value),)}
sync_adapter = _DataFrameScalarFunctionAdapter(
_resolve_udf(describe).runtime_source,
return_dtype,
True,
_UDFUsage.EXPRESSION,
"general",
)
async_adapter = _DataFrameAsyncScalarFunctionAdapter(
_resolve_udf(describe_async).runtime_source,
return_dtype,
True,
_UDFUsage.EXPRESSION,
"general",
)
sync_adapter.open(object())
async_adapter.open(object())
expected = Row(value=3, labels=["3"])
self.assertEqual(sync_adapter.eval(3), expected)
self.assertEqual(asyncio.run(async_adapter.eval(3)), expected)
def identity(value):
return value
leaf_adapter = _DataFrameScalarFunctionAdapter(
_resolve_udf(identity).runtime_source,
pf.DataType.int64(),
True,
_UDFUsage.EXPRESSION,
"general",
)
leaf_adapter.open(object())
self.assertIs(leaf_adapter._invocation(), identity)
def test_scalar_function_lifecycle_and_cleanup(self):
from pyflink.dataframe.udf import (
_DataFrameAsyncScalarFunctionAdapter,
_DataFrameScalarFunctionAdapter,
_UDFUsage,
_resolve_udf,
)
events = []
def create_adapter(source, deterministic=True, async_mode=False):
adapter_type = (
_DataFrameAsyncScalarFunctionAdapter
if async_mode
else _DataFrameScalarFunctionAdapter
)
return adapter_type(
_resolve_udf(source).runtime_source,
pf.DataType.int64(),
deterministic,
_UDFUsage.EXPRESSION,
"general",
)
class LifecycleFunction(ScalarFunction):
def __init__(self):
events.append("init")
def open(self, function_context):
events.append(("open", function_context))
def eval(self, value):
return value + 1
def close(self):
events.append("close")
context = object()
adapter = create_adapter(LifecycleFunction)
self.assertEqual(events, [])
with self.assertRaisesRegex(RuntimeError, "before open"):
adapter.eval(1)
adapter.open(context)
self.assertEqual(adapter.eval(1), 2)
adapter.close()
with self.assertRaisesRegex(RuntimeError, "before open"):
adapter.eval(1)
adapter.open(context)
self.assertEqual(adapter.eval(2), 3)
adapter.close()
self.assertEqual(
events,
[
"init",
("open", context),
"close",
"init",
("open", context),
"close",
],
)
failed_lifecycle_events = []
class NonDeterministicFunction(ScalarFunction):
def __init__(self):
failed_lifecycle_events.append("init")
def eval(self, value):
return value
def is_deterministic(self):
return False
def close(self):
failed_lifecycle_events.append("close")
mismatched_adapter = create_adapter(NonDeterministicFunction)
with self.assertRaisesRegex(ValueError, "Inconsistent deterministic"):
mismatched_adapter.open(context)
mismatched_adapter.close()
self.assertEqual(failed_lifecycle_events, ["init"])
async_events = []
class AsyncLifecycleFunction(AsyncScalarFunction):
def __init__(self):
async_events.append("init")
def open(self, function_context):
async_events.append(("open", function_context))
async def eval(self, value):
return value + 1
def close(self):
async_events.append("close")
async_adapter = create_adapter(
AsyncLifecycleFunction,
async_mode=True,
)
self.assertEqual(async_events, [])
async_adapter.open(context)
self.assertEqual(asyncio.run(async_adapter.eval(1)), 2)
async_adapter.close()
self.assertEqual(async_events, ["init", ("open", context), "close"])
initialization_failure_events = []
class ConstructorFailureFunction(ScalarFunction):
def __init__(self):
initialization_failure_events.append("init")
raise RuntimeError("constructor failed")
def eval(self, value):
return value
constructor_failure_adapter = create_adapter(ConstructorFailureFunction)
with self.assertRaisesRegex(RuntimeError, "constructor failed"):
constructor_failure_adapter.open(context)
constructor_failure_adapter.close()
self.assertEqual(initialization_failure_events, ["init"])
class OpenFailureFunction(ScalarFunction):
def __init__(self):
initialization_failure_events.append("second init")
def open(self, function_context):
initialization_failure_events.append("open")
raise RuntimeError("open failed")
def eval(self, value):
return value
def close(self):
initialization_failure_events.append("close")
open_failure_adapter = create_adapter(OpenFailureFunction)
with self.assertRaisesRegex(RuntimeError, "open failed"):
open_failure_adapter.open(context)
open_failure_adapter.close()
self.assertEqual(
initialization_failure_events,
["init", "second init", "open"],
)
deferred_constructor_calls = []
class DeferredCallable:
def __init__(self):
deferred_constructor_calls.append("init")
def __call__(self, value):
return value + 1
deferred_adapter = create_adapter(DeferredCallable)
deferred_adapter.open(context)
self.assertEqual(deferred_adapter.eval(1), 2)
deferred_adapter.close()
deferred_adapter.open(context)
self.assertEqual(deferred_adapter.eval(2), 3)
deferred_adapter.close()
self.assertEqual(deferred_constructor_calls, ["init", "init"])
class FailingCloseFunction(ScalarFunction):
def eval(self, value):
return value
def close(self):
raise RuntimeError("close failed")
failing_adapter = create_adapter(FailingCloseFunction())
failing_adapter.open(context)
with self.assertRaisesRegex(RuntimeError, "close failed"):
failing_adapter.close()
with self.assertRaisesRegex(RuntimeError, "before open"):
failing_adapter.eval(1)
def test_binding_failure_closes_and_resets_deferred_scalar_class(self):
from pyflink.dataframe.udf import (
_DataFrameScalarFunctionAdapter,
_UDFUsage,
_resolve_udf,
)
events = []
class BindingFailureFunction(ScalarFunction):
def __init__(self):
events.append("init")
def open(self, function_context):
events.append("open")
def eval(self, value):
return value
def close(self):
events.append("close")
raise RuntimeError("close failed")
adapter = _DataFrameScalarFunctionAdapter(
_resolve_udf(BindingFailureFunction).runtime_source,
pf.DataType.int64(),
True,
_UDFUsage.MAP,
"general",
)
for _ in range(2):
with self.assertRaisesRegex(ValueError, "input schema"):
adapter.open(object())
adapter.close()
self.assertEqual(
events,
["init", "open", "close", "init", "open", "close"],
)
class DataFrameUDFPlannerTests(PyFlinkDataFrameUTTestCase):
def test_batch_rejects_explicit_concurrency(self):
from pyflink.table import EnvironmentSettings, ExplainDetail, TableEnvironment
previous_environment = pf.get_table_environment()
self.addCleanup(pf.set_table_environment, previous_environment)
pf.set_table_environment(TableEnvironment.create(EnvironmentSettings.in_batch_mode()))
dataframe = pf.from_records([(1,)], schema=["value"])
identity = pf.udf(
lambda value: value, return_dtype=pf.DataType.int64(), concurrency=2)
with self.assertRaisesRegex(Exception, "concurrency is not supported in batch mode"):
dataframe.select(identity(pf.col("value"))).to_table().explain(
ExplainDetail.JSON_EXECUTION_PLAN)
def test_map_function_classes_require_explicit_custom_determinism(self):
class DefaultDeterminism(ScalarFunction):
def eval(self, value):
return {"value": value["value"]}
class CustomDeterminism(DefaultDeterminism):
def is_deterministic(self):
return False
class InheritedDeterminism(CustomDeterminism):
pass
class DeferredDeterminism(CustomDeterminism):
def __init__(self):
raise AssertionError("Function classes must be constructed on the worker.")
dataframe = pf.from_records([(1,)], schema=["value"])
dtype = pf.DataType.struct({"value": pf.DataType.int64()})
for mode in ("general", "pandas", "arrow"):
def bind(source, **kwargs):
return (dataframe.map(source, **kwargs) if mode == "general" else
dataframe.map_batches(source, batch_format=mode, **kwargs))
for source in (CustomDeterminism, InheritedDeterminism, DeferredDeterminism):
with self.subTest(mode=mode, source=source):
with self.assertRaisesRegex(TypeError, "deterministic=.*instance"):
bind(source, return_dtype=dtype)
declared = pf.udf(source, return_dtype=dtype, func_type=mode,
deterministic=False)
self.assert_dataframe_schema(bind(declared), ["value"])
for source in (DefaultDeterminism, CustomDeterminism()):
self.assert_dataframe_schema(bind(source, return_dtype=dtype), ["value"])
class CustomAsyncDeterminism(AsyncScalarFunction):
async def eval(self, value):
return {"value": value["value"]}
def is_deterministic(self):
return False
class InheritedAsyncDeterminism(CustomAsyncDeterminism):
pass
for source in (CustomAsyncDeterminism, InheritedAsyncDeterminism):
with self.assertRaisesRegex(TypeError, "deterministic=.*instance"):
dataframe.map(source, return_dtype=dtype)
declared = pf.udf(source, return_dtype=dtype, deterministic=False)
self.assert_dataframe_schema(dataframe.map(declared), ["value"])
self.assert_dataframe_schema(dataframe.map(source(), return_dtype=dtype), ["value"])
def test_map_batches_requires_matching_format_and_struct_schema(self):
dataframe = pf.from_records([(1,)], schema=["id"])
dtype = pf.DataType.struct({"value": pf.DataType.int64()})
def transform(batch):
return {"value": batch["id"]}
for source in (transform, pf.udf(transform, return_dtype=dtype, func_type="pandas")):
result = dataframe.map_batches(
source, batch_format="pandas",
**({"return_dtype": dtype} if source is transform else {}),
)
self.assertEqual(result.columns, ["value"])
for call, error, message in (
(lambda: dataframe.map_batches(transform), TypeError, "batch_format"),
(lambda: dataframe.map_batches(transform, batch_format="numpy"), ValueError, "format"),
(lambda: dataframe.map_batches(transform, batch_format="pandas"), TypeError,
"return_dtype"),
(lambda: dataframe.map_batches(transform, batch_format="pandas", return_dtype=int),
TypeError, "struct"),
(lambda: dataframe.map_batches(
pf.udf(transform, return_dtype=dtype), batch_format="pandas"), ValueError, "mode"),
):
with self.subTest(message=message):
with self.assertRaisesRegex(error, message):
call()
for field_type in (
pf.DataType.struct({"inner": pf.DataType.struct({"v": pf.DataType.int64()})}),
pf.DataType.struct({"ts": pf.DataType.timestamp_ltz(3)}),
pf.DataType.list(pf.DataType.struct({"v": pf.DataType.int64()})),
pf.DataType.list(pf.DataType.timestamp_ltz(3)),
):
with self.subTest(field_type=field_type):
output_type = pf.DataType.struct({"value": field_type})
with self.assertRaises((TypeError, ValueError)):
dataframe.map_batches(transform, batch_format="pandas",
return_dtype=output_type)
nested = dataframe.with_columns(value=pf.udf(
lambda value: value, return_dtype=field_type)(pf.col("id"))).select("value")
with self.assertRaises((TypeError, ValueError)):
nested.map_batches(transform, batch_format="pandas", return_dtype=dtype)
self.assertEqual(dataframe.map_batches(
transform, batch_format="arrow", return_dtype=output_type).columns, ["value"])
def test_map_infers_schema_and_rejects_conflicting_declarations(self):
class Output(TypedDict):
label: str
value: int
def transform(row) -> Output:
return {"value": row["id"] + 1, "label": "mapped"}
dataframe = pf.from_records([(1,)], schema=["id"])
declaration = pf.udf(transform)
for source in (transform, declaration):
result = dataframe.map(source)
self.assertEqual(result.columns, ["label", "value"])
with self.assertRaisesRegex(ValueError, "return_dtype"):
dataframe.map(declaration, return_dtype=Output)
with self.assertRaisesRegex(TypeError, "struct"):
dataframe.map(lambda row: row["id"], return_dtype=int)
with self.assertRaisesRegex(ValueError, "mode"):
dataframe.map(pf.udf(transform, return_dtype=Output, func_type="pandas"))
def test_arrow_calls_require_a_column_argument(self):
from pyflink.table import ExplainDetail
dataframe = pf.from_records([(1,)], schema=["id"])
identity = pf.udf(lambda *values: values[0], return_dtype=pf.DataType.int64(),
func_type="arrow")
for args in ((), (1,), (pf.lit(1),), (identity(),)):
with self.subTest(args=args):
with self.assertRaisesRegex(Exception, "at least one column-valued argument"):
result = dataframe.with_columns(
valid=identity(pf.col("id")), invalid=identity(*args))
result.to_table().explain(ExplainDetail.JSON_EXECUTION_PLAN)
def test_execution_resource_options_reach_java_functions(self):
@pf.udf(concurrency=2)
def add_one(value: int) -> int:
return value + 1
@pf.udf(concurrency=3)
async def async_add_one(value: int) -> int:
return value + 1
@pf.udf(
return_dtype=pf.DataType.int64(),
func_type="pandas",
concurrency=4,
batch_size=64,
)
def pandas_add_one(values: pd.Series) -> pd.Series:
return values + 1
@pf.udf(
return_dtype=pf.DataType.int64(),
func_type="arrow",
concurrency=5,
batch_size=128,
)
def arrow_add_one(values: pa.Array) -> pa.Array:
return values
cases = [
(add_one, 2, -1),
(async_add_one, 3, -1),
(pandas_add_one, 4, 64),
(arrow_add_one, 5, 128),
(pf.udf(lambda value: value, return_dtype=pf.DataType.int64(),
func_type="arrow", concurrency=2147483647, batch_size=2147483647),
2147483647, 2147483647),
]
async def default_async(value):
return value
for function in (lambda value: value, default_async):
cases.append((pf.udf(function, return_dtype=pf.DataType.int64()), -1, -1))
for declaration, expected_concurrency, expected_batch_size in cases:
with self.subTest(declaration=declaration.__name__):
java_function = (
declaration._table_udf_wrapper._java_user_defined_function()
)
self.assertEqual(
java_function.getParallelism(), expected_concurrency
)
self.assertEqual(
java_function.getMaxArrowBatchSize(), expected_batch_size
)
def test_with_columns_binds_expressions_and_resolves_output_schema(self):
@pf.udf(name="render_value")
def render(value: int, suffix: str) -> str:
return f"{value}{suffix}"
@pf.udf(
return_dtype=pf.DataType.struct(
{
"value": pf.DataType.int64(),
"tags": pf.DataType.list(pf.DataType.string()),
}
)
)
def describe(value):
return {"value": value, "tags": [str(value)]}
result = pf.from_records([(1,)], schema=["id"]).with_columns(
rendered=render(pf.col("id"), "-literal"),
description=describe(pf.col("id")),
)
self.assert_dataframe_schema(
result,
["id", "rendered", "description"],
[
TableDataTypes.BIGINT(),
TableDataTypes.STRING().not_null(),
TableDataTypes.ROW(
[
TableDataTypes.FIELD("value", TableDataTypes.BIGINT()),
TableDataTypes.FIELD(
"tags", TableDataTypes.ARRAY(TableDataTypes.STRING())
),
]
),
],
)
class DataFrameUDFITCase(PyFlinkStreamDataFrameTestCase):
def test_row_maps_preserve_literal_and_fused_inputs(self):
self.env.set_parallelism(1)
def identity(row):
return row
async def async_identity(row):
return row
@pf.udf
def increment(value: int) -> int:
return value + 1
source = pf.from_records([(1,), (2,)], schema=["id"])
tagged = source.with_columns(tag=pf.lit("xyz"))
for dataframe, fields, expected in (
(tagged.select("tag"), {"tag": pf.DataType.string()},
[Row("xyz"), Row("xyz")]),
(tagged, {"id": pf.DataType.int64(), "tag": pf.DataType.string()},
[Row(1, "xyz"), Row(2, "xyz")]),
(source.select(increment(pf.col("id")).alias("first"),
increment(pf.col("id")).alias("second")),
{"first": pf.DataType.int64(), "second": pf.DataType.int64()},
[Row(2, 2), Row(3, 3)]),
):
dtype = pf.DataType.struct(fields)
for function in (identity, async_identity):
with self.subTest(fields=fields, function=function):
result = dataframe.map(function, return_dtype=dtype).map(
identity, return_dtype=dtype)
self.assertEqual(result.collect(), expected)
def test_batch_maps_broadcast_literal_columns(self):
self.env.set_parallelism(1)
self.t_env.get_config().set("python.fn-execution.bundle.size", "3")
self.t_env.get_config().set("python.fn-execution.arrow.batch.size", "2")
dtype = pf.DataType.struct({
"tag": pf.DataType.string(), "empty": pf.DataType.int64(),
"enabled": pf.DataType.bool(), "id": pf.DataType.int64()})
dataframe = pf.from_records([(1,), (2,), (3,)], schema=["id"]).with_columns(
tag=pf.lit("x"), empty=pf.lit(None, pf.DataType.int64()),
enabled=pf.lit(True)).select("tag", "empty", "enabled", "id")
for batch_format in ("pandas", "arrow"):
with self.subTest(batch_format=batch_format):
result = dataframe.map_batches(
lambda batch: batch, batch_format=batch_format, return_dtype=dtype)
self.assertEqual(result.collect(),
[Row("x", None, True, value) for value in (1, 2, 3)])
def test_pandas_output_alignment_preserves_input_order_and_ltz(self):
import datetime
self.env.set_parallelism(1)
self.t_env.get_config().set("python.fn-execution.bundle.size", "3")
self.t_env.get_config().set("python.fn-execution.arrow.batch.size", "2")
timestamp = datetime.datetime(2020, 1, 1, 12)
def transform(batch):
return {"score": (batch["id"] * 10).sort_index(ascending=False),
"id": batch["id"], "ts": batch["ts"]}
timezone = self.t_env.get_config().get_local_timezone()
try:
for zone, epoch in [("UTC", 1577880000000), ("Asia/Shanghai", 1577851200000)]:
with self.subTest(timezone=zone):
self.t_env.get_config().set_local_timezone(zone)
from pyflink.table.expressions import call_sql, col
source = self.t_env.from_elements(
[(10, epoch), (20, epoch), (30, None)], ["id", "epoch"])
dataframe = pf.from_table(source.select(
col("id"), call_sql("TO_TIMESTAMP_LTZ(epoch, 3)").alias("ts")))
result = dataframe.map_batches(
transform, batch_format="pandas", return_dtype=pf.DataType.struct({
"score": pf.DataType.int64(), "id": pf.DataType.int64(),
"ts": pf.DataType.timestamp_ltz(3)}))
result = result.select(pf.col("score"), pf.col("id"),
pf.col("ts").cast(pf.DataType.timestamp(3)).alias("ts"))
self.assertEqual(result.collect(), [
Row(100, 10, timestamp), Row(200, 20, timestamp), Row(300, 30, None)])
finally:
self.t_env.get_config().set_local_timezone(timezone)
def test_pandas_map_preserves_row_columns_across_execution_boundaries(self):
from pyflink.table.udf import udf as table_udf
self.env.set_parallelism(1)
self.t_env.get_config().set("python.fn-execution.bundle.size", "3")
self.t_env.get_config().set("python.fn-execution.arrow.batch.size", "2")
payload_type = pf.DataType.struct({
"lookup": pf.DataType.map(pf.DataType.string().not_null(), pf.DataType.int64()),
"v": pf.DataType.int64(),
})
input_type = pf.DataType.struct({"id": pf.DataType.int64(), "payload": payload_type})
value_type = pf.DataType.struct({"v": pf.DataType.int64()})
output_type = pf.DataType.struct({"total": pf.DataType.int64(), "value": value_type})
flat_type = pf.DataType.struct({"total": pf.DataType.int64(), "v": pf.DataType.int64()})
def produce(values):
rows = [
{"lookup": [("long-key", int(value))], "v": 11} if value > 0 else
{"lookup": [], "v": None} if value == 0 else
{"lookup": None, "v": None}
for value in values
]
return pd.DataFrame({
label: pd.Series([row[field] for row in rows], index=values.index[::-1],
dtype=object)
for label, field in zip(("unrelated1", "unrelated2"), ("lookup", "v"))
})
def consume(batch):
assert isinstance(batch["payload"], pd.DataFrame)
return {"total": batch["payload"]["lookup"].map(
lambda pairs: -1 if pairs is None else sum(value for _, value in pairs)
) + batch["id"] * 10, "value": batch["payload"][["v"]].sort_index(ascending=False)}
def extract(batch):
return {"total": batch["total"], "v": batch["value"].field("v")}
@table_udf(result_type=flat_type._to_table_data_type(), func_type="pandas")
def table_extract(batch):
assert isinstance(batch["total"], pd.Series)
return pd.DataFrame({name: batch[name] for name in ("total", "v")})
producer = pf.udf(produce, return_dtype=payload_type, func_type="pandas")
for transported in (False, True):
with self.subTest(transported=transported):
dataframe = pf.from_records([(1,), (0,), (-1,)], schema=["id"]).with_columns(
payload=producer(pf.col("id")))
if transported:
dataframe = dataframe.map_batches(
lambda batch: batch, return_dtype=input_type, batch_format="arrow")
mapped = dataframe.map_batches(
consume, return_dtype=output_type, batch_format="pandas").map_batches(
lambda batch: batch, return_dtype=output_type, batch_format="pandas")
for table_map in (False, True):
result = (pf.from_table(mapped.to_table().map(table_extract)) if table_map
else mapped.map_batches(extract, batch_format="arrow",
return_dtype=flat_type))
self.assertEqual(sorted(result.collect(), key=lambda row: row[0]),
[Row(-11, None), Row(0, None), Row(11, 11)])
def test_row_and_batch_maps_in_one_job(self):
self.env.set_parallelism(1)
self.t_env.get_config().set("python.fn-execution.bundle.size", "3")
self.t_env.get_config().set("python.fn-execution.arrow.batch.size", "2")
dtype = pf.DataType.struct({"id": pf.DataType.int64(), "value": pf.DataType.int64()})
class Prepare(ScalarFunction):
def eval(self, row):
assert isinstance(row, dict)
return {"value": row["id"] * 2, "id": row["id"], "ignored": 0}
def is_deterministic(self):
return False
prepare = pf.udf(Prepare, return_dtype=dtype, deterministic=False)
async def enrich(row):
assert isinstance(row, dict)
return {"id": row["id"], "value": row["value"] + 1}
def pandas_add(batch):
assert isinstance(batch, dict)
assert all(isinstance(column, pd.Series) for column in batch.values())
assert 1 <= len(batch["id"]) <= 2
return {"id": batch["id"], "value": batch["value"] + 10}
def arrow_double(batch):
import pyarrow.compute as pc
assert isinstance(batch, dict)
assert all(isinstance(column, pa.Array) for column in batch.values())
return {"id": pa.chunked_array([batch["id"].slice(0, 1), batch["id"].slice(1)]),
"value": pc.multiply(batch["value"], 2)}
flat_dtype = pf.DataType.struct({
"id": pf.DataType.int64(), "value": pf.DataType.int64(),
"inner": pf.DataType.int64(),
})
nested_dtype = pf.DataType.struct({
"id": pf.DataType.int64(),
"payload": pf.DataType.struct({
"value": pf.DataType.int64(),
"inner": pf.DataType.struct({"value": pf.DataType.int64()}),
}),
})
def pandas_prepare(batch):
return {"id": batch["id"], "value": batch["value"],
"inner": pd.Series([None if identifier == 3 else 3
for identifier in batch["id"]], dtype=object)}
def arrow_nest(batch):
return {"id": batch["id"], "payload": pa.array([
None if identifier == 5 else {"value": value, "inner":
None if identifier == 4 else {"value": inner}}
for identifier, value, inner in zip(
batch["id"].to_pylist(), batch["value"].to_pylist(), batch["inner"].to_pylist())
], type=pa.struct([("value", pa.int64()),
("inner", pa.struct([("value", pa.int64())]))]))}
def arrow_unnest(batch):
assert isinstance(batch["payload"], pa.StructArray)
return {"id": batch["id"], "value": pa.array([
-1 if payload is None else -2 if payload["inner"] is None else -3
if payload["inner"]["value"] is None
else payload["value"] + payload["inner"]["value"]
for payload in batch["payload"].to_pylist()
], type=pa.int64())}
result = (pf.from_records([(1,), (2,), (3,), (4,), (5,)], schema=["id"])
.map(prepare).map(enrich, return_dtype=dtype)
.map_batches(pandas_add, return_dtype=dtype, batch_format="pandas")
.map_batches(arrow_double, return_dtype=dtype, batch_format="arrow")
.map_batches(pandas_prepare, return_dtype=flat_dtype, batch_format="pandas")
.map_batches(arrow_nest, return_dtype=nested_dtype, batch_format="arrow")
.map_batches(arrow_unnest, return_dtype=dtype, batch_format="arrow"))
self.assertEqual(sorted(result.collect(), key=lambda row: row[0]),
[Row(1, 29), Row(2, 33), Row(3, -3), Row(4, -2), Row(5, -1)])
def test_supported_scalar_udfs_in_one_job(self):
import pyarrow.compute as pc
self.env.set_parallelism(1)
self.t_env.get_config().set("python.fn-execution.bundle.size", "3")
self.t_env.get_config().set("python.fn-execution.arrow.batch.size", "2")
@pf.udf(return_dtype=pf.DataType.string())
def normalize_name(names: pa.Array) -> pa.Array:
return pc.utf8_upper(names)
@pf.udf(return_dtype=pf.DataType.struct({"value": pf.DataType.int64().not_null()}))
def describe(values: pa.Array) -> pa.ChunkedArray:
result = pa.StructArray.from_arrays([pc.multiply(values, 2)], names=["value"])
return pa.chunked_array([result.slice(0, 1), result.slice(1)])
@pf.udf(return_dtype=pf.DataType.int64())
def struct_value(values: pa.Array) -> pa.Array:
return pc.struct_field(values, "value")
@dataclass
class Details:
doubled: int
labels: list
@pf.udf
async def add_two(value: int) -> int:
return value + 2
@pf.udf(return_dtype=pf.DataType.int64(), func_type="pandas")
def add_three(values: pd.Series) -> pd.Series:
return values + 3
@pf.udf(
return_dtype=pf.DataType.struct(
{
"doubled": pf.DataType.int64(),
"labels": pf.DataType.list(pf.DataType.string()),
}
)
)
def details(value):
return Details(doubled=value * 2, labels=[str(value)])
class DeferredCallable:
def __call__(self, value: int) -> int:
return value + 4
class OpenedScalarFunction(ScalarFunction):
def open(self, function_context):
self._increment = 5
def eval(self, *values: int) -> int:
value, = values
return value + self._increment
deferred = pf.udf(DeferredCallable)
opened_scalar_class = pf.udf(OpenedScalarFunction)
result = (
pf.from_records([(1, "alice"), (2, None), (3, "Bob")], schema=["id", "name"])
.with_columns(async_value=add_two(pf.col("id")))
.with_columns(
pandas_value=add_three(pf.col("id")),
details=details(pf.col("id")),
deferred_value=deferred(pf.col("id")),
scalar_value=opened_scalar_class(pf.col("id")),
normalized_name=normalize_name(pf.col("name")),
arrow_details=describe(pf.col("id")),
arrow_after_pandas=struct_value(describe(add_three(pf.col("id")))),
pandas_after_arrow=add_three(struct_value(describe(pf.col("id")))),
)
)
self.assertEqual(
sorted(result.collect(), key=lambda row: row[0]),
[
Row(1, "alice", 3, 4, Row(2, ["1"]), 5, 6, "ALICE", Row(2), 8, 5),
Row(2, None, 4, 5, Row(4, ["2"]), 6, 7, None, Row(4), 10, 7),
Row(3, "Bob", 5, 6, Row(6, ["3"]), 7, 8, "BOB", Row(6), 12, 9),
],
)
if __name__ == "__main__":
unittest.main()