| ################################################################################ | |
| # Licensed to the Apache Software Foundation (ASF) under one | |
| # or more contributor license agreements. See the NOTICE file | |
| # distributed with this work for additional information | |
| # regarding copyright ownership. The ASF licenses this file | |
| # to you under the Apache License, Version 2.0 (the | |
| # "License"); you may not use this file except in compliance | |
| # with the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| ################################################################################ | |
| import functools | |
| import os | |
| import threading | |
| import unittest | |
| from dataclasses import replace | |
| from typing import ( | |
| Annotated, Any, Dict, Generator, Iterable, Iterator, List, Mapping, MutableMapping, | |
| Optional, Tuple, TypedDict, Union, | |
| ) | |
| import cloudpickle | |
| from py4j.protocol import Py4JJavaError | |
| import pyflink.dataframe as pf | |
| from pyflink.common import Row | |
| from pyflink.dataframe.udtf import _resolve_flat_map_udtf | |
| from pyflink.table import DataTypes | |
| from pyflink.table.expression import Expression | |
| from pyflink.table.expressions import call as table_call | |
| from pyflink.table.udf import ScalarFunction, TableFunction, udtf as table_udtf | |
| from pyflink.testing.test_case_utils import ( | |
| PyFlinkBatchTableTestCase, | |
| PyFlinkDataFrameUTTestCase, | |
| PyFlinkStreamDataFrameTestCase, | |
| ) | |
| class _Output(TypedDict): | |
| value: int | |
| label: str | |
| def _eval_udtf(declaration, *args): | |
| adapter = declaration._create_table_wrapper()._func | |
| adapter.open(None) | |
| try: | |
| return list(adapter.eval(*args)) | |
| finally: | |
| adapter.close() | |
| class DataFrameUDTFDeclarationTests(unittest.TestCase): | |
| def test_missing_row_fields_remain_nullable_in_expression_results(self): | |
| details = pf.DataType.struct({"label": pf.DataType.string()}) | |
| dtype = pf.DataType.struct({ | |
| "missing": pf.DataType.string(), | |
| "details": details, | |
| "items": pf.DataType.list(details), | |
| }) | |
| declaration = pf.udtf(lambda value: [{"details": {}, "items": [{}]}], | |
| return_dtype=dtype) | |
| self.assertEqual(_eval_udtf(declaration, 1), [ | |
| Row(missing=None, details=Row(label=None), items=[Row(label=None)])]) | |
| def test_decorator_forms_and_type_inference(self): | |
| def emit(value) -> Iterator[_Output]: | |
| yield {"value": value, "label": "ok"} | |
| expected = pf.DataType.struct({ | |
| "value": pf.DataType.int64().not_null(), | |
| "label": pf.DataType.string().not_null(), | |
| }).not_null() | |
| for declaration in (pf.udtf(emit), pf.udtf()(emit), pf.udtf(return_dtype=expected)(emit), | |
| pf.udtf(emit, return_dtype=_Output)): | |
| self.assertEqual(declaration.return_dtype, expected) | |
| self.assertEqual(declaration.__name__, "emit") | |
| def test_iterable_annotations_describe_one_emitted_item(self): | |
| hints = (Iterator[int], Iterable[int], Generator[int, None, None], List[int], list[int]) | |
| for hint in hints: | |
| def emit(value): | |
| return [value] | |
| emit.__annotations__ = {"return": hint} | |
| with self.subTest(hint=hint): | |
| self.assertEqual(pf.udtf(emit).return_dtype, pf.DataType.int64().not_null()) | |
| def test_explicit_type_does_not_resolve_annotations(self): | |
| def emit(value): | |
| return [value] | |
| emit.__annotations__ = {"value": "UnavailableInput", "return": "UnavailableOutput"} | |
| self.assertEqual( | |
| pf.udtf(emit, return_dtype=int).return_dtype, pf.DataType.int64().not_null()) | |
| emit.__annotations__["return"] = Iterator[int] | |
| self.assertEqual(pf.udtf(emit).return_dtype, pf.DataType.int64().not_null()) | |
| def test_callable_instance_and_partial(self): | |
| class Repeat: | |
| def __init__(self, count): | |
| self.count = count | |
| def __call__(self, value) -> Iterator[int]: | |
| yield from [value] * self.count | |
| def repeat(count, value) -> Iterator[int]: | |
| yield from [value] * count | |
| for func in (Repeat(2), functools.partial(repeat, 2)): | |
| self.assertEqual(_eval_udtf(pf.udtf(func), 3), [Row(3), Row(3)]) | |
| def test_invalid_declarations(self): | |
| class Scalar(ScalarFunction): | |
| def eval(self, value): | |
| return value | |
| class NotCallable: | |
| pass | |
| class RequiresCallableArgument: | |
| def __init__(self, count): | |
| self.count = count | |
| def __call__(self, value): | |
| return [value] * self.count | |
| class RequiresArgument(TableFunction): | |
| def __init__(self, value): | |
| self.value = value | |
| def eval(self, value): | |
| return [value] | |
| for func in (123, Scalar(), Scalar, NotCallable, | |
| RequiresCallableArgument, RequiresArgument): | |
| with self.subTest(func=func), self.assertRaises(TypeError): | |
| pf.udtf(func, return_dtype=int) | |
| for hint in (Iterator, Iterator[Tuple[int, ...]], Iterator[Tuple]): | |
| def emit(row): | |
| return [] | |
| emit.__annotations__ = {"return": hint} | |
| with self.subTest(hint=hint), self.assertRaises(TypeError): | |
| pf.udtf(emit) | |
| with self.assertRaisesRegex(TypeError, "return_dtype"): | |
| pf.udtf(lambda row: []) | |
| with self.assertRaisesRegex(ValueError, "at least one"): | |
| pf.udtf(lambda row: [], return_dtype=pf.DataType.struct({})) | |
| def test_async_functions_are_rejected(self): | |
| async def coroutine(row): | |
| return [row] | |
| async def generator(row): | |
| yield row | |
| class AsyncCallable: | |
| async def __call__(self, row): | |
| yield row | |
| @functools.wraps(generator) | |
| def hidden_generator(row): | |
| return generator(row) | |
| for func in (coroutine, generator, AsyncCallable, AsyncCallable(), hidden_generator): | |
| with self.subTest(func=func), self.assertRaisesRegex(TypeError, "async"): | |
| pf.udtf(func, return_dtype=int) | |
| def test_names_follow_table_udtf_defaults(self): | |
| def emit(row) -> Iterator[int]: | |
| yield row[0] | |
| class Expand(TableFunction): | |
| def eval(self, row) -> Iterator[int]: | |
| yield row[0] | |
| class Repeat: | |
| def __call__(self, row) -> Iterator[int]: | |
| yield row[0] | |
| cases = ( | |
| (emit, None, "emit"), (Expand, None, "Expand"), (Expand(), None, "Expand"), | |
| (Repeat(), None, "Repeat"), (emit, "", "emit"), (emit, "custom_name", "custom_name"), | |
| ) | |
| for func, name, expected in cases: | |
| with self.subTest(func=func, name=name): | |
| declaration = pf.udtf(func, name=name) | |
| self.assertEqual(declaration.__name__, expected) | |
| self.assertEqual(declaration._create_table_wrapper()._name, expected) | |
| def test_determinism_validation(self): | |
| with self.assertRaises(TypeError): | |
| pf.udtf(lambda row: [], return_dtype=int, deterministic=1) | |
| class Random(TableFunction): | |
| def eval(self, value) -> Iterator[int]: | |
| yield value | |
| def is_deterministic(self): | |
| return False | |
| with self.assertRaisesRegex(ValueError, "Inconsistent deterministic"): | |
| pf.udtf(Random()) | |
| adapter = pf.udtf(Random, deterministic=False, name="random")._create_table_wrapper()._func | |
| adapter.open(None) | |
| try: | |
| self.assertFalse(adapter.is_deterministic()) | |
| self.assertEqual(list(adapter.eval(1)), [Row(1)]) | |
| finally: | |
| adapter.close() | |
| with self.assertRaisesRegex(ValueError, "Inconsistent deterministic"): | |
| pf.udtf(Random) | |
| def test_table_function_class_constructs_on_client(self): | |
| events = [] | |
| class Expand(TableFunction): | |
| def __init__(self): | |
| events.append("init") | |
| def open(self, context): | |
| events.append(("open", context)) | |
| def eval(self, value) -> Iterator[int]: | |
| yield value | |
| def close(self): | |
| events.append("close") | |
| declaration = pf.udtf(Expand) | |
| self.assertEqual(events, ["init"]) | |
| adapter = declaration._create_table_wrapper()._func | |
| self.assertEqual(events, ["init"]) | |
| adapter.open("worker") | |
| self.assertEqual(list(adapter.eval(7)), [Row(7)]) | |
| adapter.close() | |
| self.assertEqual(events, ["init", ("open", "worker"), "close"]) | |
| with self.assertRaisesRegex(RuntimeError, "before open"): | |
| adapter.eval(7) | |
| def test_adapter_serializes_only_worker_metadata(self): | |
| def emit(value) -> Iterator[int]: | |
| yield value | |
| declaration = pf.udtf(emit) | |
| context = replace(declaration._declaration_context, | |
| localns={"client_lock": threading.Lock()}) | |
| declaration = replace(declaration, _declaration_context=context) | |
| adapter = cloudpickle.loads(cloudpickle.dumps(declaration._create_table_wrapper()._func)) | |
| adapter.open(None) | |
| try: | |
| self.assertEqual(list(adapter.eval(5)), [Row(5)]) | |
| finally: | |
| adapter.close() | |
| def test_table_function_constructor_failure_is_reported_on_client(self): | |
| class FailingConstructor(TableFunction): | |
| def __init__(self): | |
| raise RuntimeError("constructor failed") | |
| def eval(self, value) -> Iterator[int]: | |
| yield value | |
| with self.assertRaisesRegex(RuntimeError, "constructor failed"): | |
| pf.udtf(FailingConstructor) | |
| def test_table_function_close_failure_resets_adapter(self): | |
| class FailingClose(TableFunction): | |
| def eval(self, value) -> Iterator[int]: | |
| yield value | |
| def close(self): | |
| raise RuntimeError("close failed") | |
| adapter = pf.udtf(FailingClose)._create_table_wrapper()._func | |
| adapter.open(None) | |
| self.assertEqual(list(adapter.eval(1)), [Row(1)]) | |
| with self.assertRaisesRegex(RuntimeError, "close failed"): | |
| adapter.close() | |
| with self.assertRaisesRegex(RuntimeError, "before open"): | |
| adapter.eval(1) | |
| adapter.close() | |
| def test_output_cardinality_and_lazy_iteration(self): | |
| events = [] | |
| def emit(value): | |
| events.append(value) | |
| yield value | |
| events.append(value + 1) | |
| yield value + 1 | |
| adapter = pf.udtf(emit, return_dtype=int)._create_table_wrapper()._func | |
| adapter.open(None) | |
| try: | |
| output = adapter.eval(2) | |
| self.assertEqual(events, []) | |
| self.assertEqual(next(output), Row(2)) | |
| self.assertEqual(events, [2]) | |
| self.assertEqual(list(output), [Row(3)]) | |
| finally: | |
| adapter.close() | |
| cases = [(None, []), ([], []), (3, [Row(3)]), ((3,), [Row(3)]), | |
| (Row(3), [Row(3)]), ([None, 3], [Row(None), Row(3)])] | |
| for result, expected in cases: | |
| self.assertEqual(_eval_udtf(pf.udtf(lambda: result, return_dtype=int)), expected) | |
| def test_struct_and_map_results(self): | |
| dtype = pf.DataType.struct({ | |
| "value": pf.DataType.int64(), | |
| "nested": pf.DataType.list(pf.DataType.struct({"label": pf.DataType.string()})), | |
| "attributes": pf.DataType.map(pf.DataType.string(), pf.DataType.int64()), | |
| }) | |
| result = {"attributes": {"x": 1}, "nested": [{"label": "ok"}], "value": 7} | |
| self.assertEqual(_eval_udtf(pf.udtf(lambda: result, return_dtype=dtype)), [ | |
| Row(value=7, nested=[Row(label="ok")], attributes={"x": 1})]) | |
| map_type = pf.DataType.map(pf.DataType.string(), pf.DataType.int64()) | |
| for result in ({"x": 1}, Row({"x": 1}), ({"x": 1},)): | |
| self.assertEqual(_eval_udtf(pf.udtf(lambda: result, return_dtype=map_type)), | |
| [Row({"x": 1})]) | |
| def test_runtime_errors_are_propagated(self): | |
| dtype = pf.DataType.struct({"a": pf.DataType.int64(), "b": pf.DataType.int64()}) | |
| for result, return_dtype in (((1,), dtype), ((1, 2), pf.DataType.int64())): | |
| declaration = pf.udtf(lambda: result, return_dtype=return_dtype) | |
| with self.assertRaises(ValueError): | |
| _eval_udtf(declaration) | |
| def fail(row): | |
| raise RuntimeError("user failure") | |
| with self.assertRaisesRegex(RuntimeError, "user failure"): | |
| _eval_udtf(pf.udtf(fail, return_dtype=int), Row(1)) | |
| def test_single_named_field_accepts_scalar_items(self): | |
| cases = [ | |
| (pf.DataType.string(), ["hello", "world"], [Row(word="hello"), Row(word="world")]), | |
| (pf.DataType.list(pf.DataType.int64()), [[1, 2], []], [Row(word=[1, 2]), Row(word=[])]), | |
| ] | |
| for field_type, result, expected in cases: | |
| dtype = pf.DataType.struct({"word": field_type}) | |
| self.assertEqual(_eval_udtf(pf.udtf(lambda: result, return_dtype=dtype)), expected) | |
| def test_flat_map_validates_row_input_before_building_expression(self): | |
| def column(value: int) -> Iterator[int]: | |
| yield value | |
| def multiple(left, right) -> Iterator[int]: | |
| yield left + right | |
| def keyword(row, *, required) -> Iterator[int]: | |
| yield required | |
| for func in (column, multiple, keyword, | |
| pf.udtf(column), pf.udtf(multiple), pf.udtf(keyword)): | |
| with self.subTest(func=func), self.assertRaisesRegex(ValueError, "row argument"): | |
| _resolve_flat_map_udtf(func, None, ["x"]) | |
| with self.assertRaisesRegex(ValueError, "return_dtype"): | |
| _resolve_flat_map_udtf(pf.udtf(column), int, ["x"]) | |
| with self.assertRaises(TypeError): | |
| _resolve_flat_map_udtf(None, int, ["x"]) | |
| def test_incompatible_input_annotations(self): | |
| for hint in (Row, Tuple[int], Optional[int], Union[int, str], | |
| Annotated[int, "column value"]): | |
| def expand(row) -> Iterator[int]: | |
| yield row | |
| expand.__annotations__["row"] = hint | |
| for func in (expand, pf.udtf(expand)): | |
| with self.subTest(hint=hint, func=func): | |
| with self.assertRaisesRegex(ValueError, "row argument"): | |
| _resolve_flat_map_udtf(func, None, ["x"]) | |
| def test_flat_map_passes_dict_without_mutating_input(self): | |
| received = [] | |
| @pf.udtf | |
| def expand(row: Dict[str, Any]) -> Iterator[int]: | |
| received.append(row.copy()) | |
| yield row.pop("x") | |
| adapter = expand._create_table_wrapper(("z", "x"))._func | |
| adapter.open(None) | |
| self.addCleanup(adapter.close) | |
| original = Row(original_z=7, original_x=3) | |
| for value in ((7, 3), original): | |
| self.assertEqual(list(adapter.eval(value)), [Row(3)]) | |
| self.assertEqual(received[-1], {"z": 7, "x": 3}) | |
| self.assertEqual(original.as_dict(), {"original_z": 7, "original_x": 3}) | |
| def test_callable_class_initializes_per_worker_adapter(self): | |
| events = [] | |
| class Counter: | |
| def __init__(self): | |
| events.append("init") | |
| self.count = 0 | |
| def __call__(self, value: int) -> Iterator[int]: | |
| self.count += 1 | |
| yield value + self.count | |
| declaration = pf.udtf(Counter) | |
| adapters = [declaration._create_table_wrapper()._func for _ in range(2)] | |
| self.assertEqual(events, []) | |
| for adapter in adapters: | |
| adapter.open(None) | |
| self.addCleanup(adapter.close) | |
| self.assertEqual(list(adapter.eval(10)), [Row(11)]) | |
| self.assertEqual(list(adapter.eval(10)), [Row(12)]) | |
| self.assertEqual(events, ["init", "init"]) | |
| adapters[0].close() | |
| with self.assertRaisesRegex(RuntimeError, "before open"): | |
| adapters[0].eval(10) | |
| def test_callable_class_constructor_failure_is_reported_on_worker(self): | |
| class FailingConstructor: | |
| def __init__(self): | |
| raise RuntimeError("constructor failed") | |
| def __call__(self, value) -> Iterator[int]: | |
| yield value | |
| adapter = pf.udtf(FailingConstructor)._create_table_wrapper()._func | |
| with self.assertRaisesRegex(RuntimeError, "constructor failed"): | |
| adapter.open(None) | |
| adapter.close() | |
| class DataFrameUDTFPlanningTests(PyFlinkDataFrameUTTestCase): | |
| def test_compatible_input_annotations(self): | |
| class Input(TypedDict): | |
| x: int | |
| source = pf.from_dict({"x": [1]}) | |
| for hint in (Input, Optional[Input], Optional[Dict[str, Any]], | |
| Union[int, Dict[str, Any]], Mapping[str, Any], MutableMapping[str, Any], | |
| Annotated[Dict[str, Any], "input row"]): | |
| def expand(row) -> Iterator[int]: | |
| yield row["x"] | |
| expand.__annotations__["row"] = hint | |
| for func in (expand, pf.udtf(expand)): | |
| with self.subTest(hint=hint, func=func): | |
| self.assertEqual(source.flat_map(func).columns, ["f0"]) | |
| def test_callable_class_signatures_and_annotations(self): | |
| class Expand: | |
| def __call__(self, row: Dict[str, Any], optional=0) -> Iterator[int]: | |
| yield row["x"] + optional | |
| class ClassMethod: | |
| @classmethod | |
| def __call__(cls, row: Dict[str, Any]) -> Iterator[int]: | |
| yield row["x"] | |
| class StaticMethod: | |
| @staticmethod | |
| def __call__(row: Dict[str, Any]) -> Iterator[int]: | |
| yield row["x"] | |
| class MultipleArguments: | |
| def __call__(self, left, right) -> Iterator[int]: | |
| yield left + right | |
| class ColumnArgument: | |
| def __call__(self, value: int) -> Iterator[int]: | |
| yield value | |
| source = pf.from_dict({"x": [1]}) | |
| for cls in (Expand, ClassMethod, StaticMethod): | |
| for func in (cls, pf.udtf(cls)): | |
| with self.subTest(func=func): | |
| self.assertEqual(source.flat_map(func).columns, ["f0"]) | |
| for cls in (MultipleArguments, ColumnArgument): | |
| for func in (cls, pf.udtf(cls)): | |
| with self.subTest(func=func), self.assertRaisesRegex(ValueError, "row argument"): | |
| source.flat_map(func) | |
| def test_wrapped_static_method_still_validates_input_annotations(self): | |
| def transparent(func): | |
| @functools.wraps(func) | |
| def wrapper(*args, **kwargs): | |
| return func(*args, **kwargs) | |
| return wrapper | |
| class Expand: | |
| @staticmethod | |
| @transparent | |
| def __call__(value: int) -> Iterator[int]: | |
| yield value | |
| source = pf.from_dict({"x": [1]}) | |
| for func in (Expand, pf.udtf(Expand), Expand(), pf.udtf(Expand())): | |
| with self.subTest(func=func), self.assertRaisesRegex(ValueError, "dict row argument"): | |
| source.flat_map(func) | |
| def test_unnamed_multiple_fields_require_schema(self): | |
| def emit(row) -> Iterator[Tuple[int, str]]: | |
| yield row["x"], "ok" | |
| source = pf.from_dict({"x": [1]}) | |
| with self.assertRaisesRegex(ValueError, "named output fields"): | |
| source.flat_map(emit) | |
| named = source.flat_map(emit, return_dtype="ROW<value BIGINT, label STRING>") | |
| self.assertEqual(named.columns, ["value", "label"]) | |
| def test_call_metadata_aliases_and_shared_declaration(self): | |
| @pf.udtf | |
| def emit(value) -> Iterator[_Output]: | |
| yield {"value": value, "label": "ok"} | |
| source = pf.from_dict({"x": [1]}) | |
| call = emit(pf.col("x")) | |
| aliased = call.alias("v", "l") | |
| self.assertIsNone(call.output_aliases) | |
| self.assertEqual(aliased.output_aliases, ("v", "l")) | |
| for names in (("one",), ("same", "same"), ("", "other")): | |
| with self.subTest(names=names), self.assertRaises(ValueError): | |
| call.alias(*names) | |
| table = source.to_table().join_lateral(aliased.expression) | |
| self.assertEqual(pf.from_table(table).columns, ["x", "v", "l"]) | |
| def test_constructing_plan_does_not_reconstruct_user_class(self): | |
| events = [] | |
| class Expand(TableFunction): | |
| def __init__(self): | |
| events.append("init") | |
| def eval(self, row: Dict[str, Any]) -> Iterator[int]: | |
| yield next(iter(row.values())) | |
| declaration = pf.udtf(Expand) | |
| self.assertEqual(events, ["init"]) | |
| for name in ("x", "y"): | |
| result = pf.from_dict({name: [1]}).flat_map(declaration) | |
| self.assertEqual(result.columns, ["f0"]) | |
| self.assertEqual(events, ["init"]) | |
| class DataFrameLateralJoinPlanningTests(PyFlinkDataFrameUTTestCase): | |
| def test_output_names_and_call_reuse(self): | |
| source = pf.from_dict({"x": [1]}) | |
| @pf.udtf | |
| def named(value) -> Iterator[_Output]: | |
| yield {"value": value, "label": "ok"} | |
| call = named(pf.col("x")) | |
| inferred_types = [DataTypes.BIGINT().not_null(), DataTypes.STRING().not_null()] | |
| for function_call, names, output_types in ( | |
| (call, ["value", "label"], inferred_types), | |
| (call.alias("v", "l"), ["v", "l"], inferred_types), | |
| (pf.udtf(lambda x: [(x, "ok")], return_dtype="ROW<v BIGINT, l STRING>")( | |
| pf.col("x")), ["v", "l"], [DataTypes.BIGINT(), DataTypes.STRING()]), | |
| ): | |
| with self.subTest(names=names): | |
| self.assert_dataframe_schema(source.join_lateral(function_call), ["x"] + names, | |
| [DataTypes.BIGINT()] + output_types) | |
| self.assertIsNone(call.output_aliases) | |
| self.assert_dataframe_schema(source.join_lateral(call), ["x", "value", "label"]) | |
| @pf.udtf | |
| def pair(value, label: str) -> Iterator[Tuple[int, str]]: | |
| yield value, label | |
| result = source.join_lateral(pair(pf.col("x"), pf.lit("ok")).alias("v", "l")) | |
| self.assertEqual(result.columns, ["x", "v", "l"]) | |
| def test_raw_expression_aliases(self): | |
| source = pf.from_dict({"x": [1]}) | |
| emit = table_udtf(lambda x: [x], result_types=[DataTypes.BIGINT()]) | |
| self.t_env.create_temporary_system_function("emit", emit) | |
| for expression in (emit(pf.col("x")).alias("out"), | |
| table_call("emit", pf.col("x")).alias("out")): | |
| with self.subTest(expression=str(expression)): | |
| # Alias names belong to the expression tree, not a Python wrapper. | |
| rebuilt = Expression(expression._j_expr) | |
| self.assert_dataframe_schema(source.join_lateral(rebuilt), ["x", "out"], | |
| [DataTypes.BIGINT(), DataTypes.BIGINT()]) | |
| with self.assertRaisesRegex(ValueError, "require alias"): | |
| source.join_lateral(emit(pf.col("x"))) | |
| def test_invalid_calls_and_output_names(self): | |
| source = pf.from_dict({"x": [1]}) | |
| scalar = pf.udtf(lambda x: [x], return_dtype=int) | |
| unnamed = scalar(pf.col("x")) | |
| named = pf.udtf(lambda x: [(x, "ok")], return_dtype="ROW<x BIGINT, label STRING>") | |
| raw = table_udtf(lambda x: [(x, "ok")], | |
| result_types=[DataTypes.BIGINT(), DataTypes.STRING()])(pf.col("x")) | |
| cases = ( | |
| (unnamed, ValueError, "require output names"), | |
| (unnamed.alias("x"), ValueError, "conflict"), | |
| (named(pf.col("x")), ValueError, "conflict"), | |
| (raw.alias("same", "same"), ValueError, "unique"), | |
| (raw.alias("", "label"), ValueError, "non-empty"), | |
| (raw.alias("x", "label"), ValueError, "conflict"), | |
| (None, TypeError, "table_function_call"), | |
| (scalar, TypeError, "table_function_call"), | |
| ) | |
| for function_call, error, message in cases: | |
| with self.subTest(message=message, function_call=function_call): | |
| with self.assertRaisesRegex(error, message): | |
| source.join_lateral(function_call) | |
| def test_option_types_and_predicate_validation(self): | |
| source = pf.from_dict({"x": [1]}) | |
| emit = pf.udtf(lambda x: [x], return_dtype=int)(pf.col("x")).alias("out") | |
| for kwargs in ({"ignore_empty": 0}, {"ignore_empty": None}, {"on": True}, {"on": "x"}): | |
| with self.subTest(kwargs=kwargs), self.assertRaises(TypeError): | |
| source.join_lateral(emit, **kwargs) | |
| for predicate in (pf.lit(1), pf.col("missing") > 0): | |
| with self.subTest(predicate=str(predicate)), self.assertRaisesRegex( | |
| Py4JJavaError, "ValidationException"): | |
| source.join_lateral(emit, on=predicate) | |
| def test_predicates_plan_in_stream_and_batch(self): | |
| from pyflink.datastream import RuntimeExecutionMode | |
| emit = pf.udtf(lambda x: [x], return_dtype=int) | |
| positive = pf.udf(lambda x: x > 0, return_dtype=bool) | |
| for mode in (RuntimeExecutionMode.STREAMING, RuntimeExecutionMode.BATCH): | |
| self.env.set_runtime_mode(mode) | |
| source = pf.from_dict({"x": [1]}) | |
| call = emit(pf.col("x") + 1).alias("out") | |
| for predicate in (None, pf.col("x") > 0, pf.col("out") > pf.col("x"), | |
| positive(pf.col("out"))): | |
| with self.subTest(mode=mode, predicate=str(predicate)): | |
| self.assertIn("PythonCorrelate", source.join_lateral(call, on=predicate) | |
| .to_table().explain()) | |
| for predicate in (None, pf.lit(True)): | |
| with self.subTest(mode=mode, predicate=str(predicate)): | |
| self.assertIn("PythonCorrelate", source.join_lateral( | |
| call, on=predicate, ignore_empty=False).to_table().explain()) | |
| def test_left_predicate_retains_table_api_restriction(self): | |
| source = pf.from_dict({"x": [1]}) | |
| emit = pf.udtf(lambda x: [x], return_dtype=int)(pf.col("x")).alias("out") | |
| with self.assertRaisesRegex(Py4JJavaError, "can only be empty or literal true"): | |
| source.join_lateral(emit, on=pf.col("out") > 0, ignore_empty=False) | |
| class _DataFrameUDTFTests: | |
| def setUp(self): | |
| super().setUp() | |
| previous = pf.get_table_environment() | |
| self.addCleanup(pf.set_table_environment, previous) | |
| pf.set_table_environment(self.t_env) | |
| class _DataFrameFlatMapTests(_DataFrameUDTFTests): | |
| def test_typed_dict_input_and_named_output_pipeline(self): | |
| class Input(TypedDict): | |
| count: int | |
| label: str | |
| def expand(row: Input) -> Iterator[_Output]: | |
| for value in range(row["count"]): | |
| yield {"label": row["label"], "value": value} | |
| source = pf.from_records([(0, "empty"), (1, "a"), (2, "b")], schema=["count", "label"]) | |
| result = source.flat_map(expand) | |
| self.assertEqual(result.columns, ["value", "label"]) | |
| self.assertEqual(result.schema.get_field_data_types(), | |
| [DataTypes.BIGINT().not_null(), DataTypes.STRING().not_null()]) | |
| rows = result.filter(pf.col("value") >= 0).select("label", "value").collect() | |
| self.assertCountEqual(rows, [Row("a", 0), Row("b", 0), Row("b", 1)]) | |
| def test_plain_and_decorated_functions_receive_dict(self): | |
| dtype = pf.DataType.struct({ | |
| "is_dict": pf.DataType.bool(), | |
| "first": pf.DataType.int64(), | |
| "by_name": pf.DataType.int64(), | |
| }) | |
| def expand(row: Dict[str, Any]): | |
| yield isinstance(row, dict), next(iter(row.values())), row["x"] | |
| source = pf.from_records([(7, 3)], schema=["z", "x"]) | |
| plain = source.flat_map(expand, return_dtype=dtype) | |
| decorated = source.flat_map(pf.udtf(expand, return_dtype=dtype)) | |
| self.assertEqual(plain.columns, ["is_dict", "first", "by_name"]) | |
| self.assertEqual(decorated.columns, plain.columns) | |
| self.assertEqual(plain.union_all(decorated).collect(), [Row(True, 7, 3)] * 2) | |
| def test_wrapper_row_input_reuse_and_column_calls(self): | |
| @pf.udtf | |
| def expand(value) -> Iterator[int]: | |
| value = next(iter(value.values())) if isinstance(value, dict) else value | |
| yield value | |
| yield value + 1 | |
| a = pf.from_dict({"x": [1]}) | |
| b = pf.from_dict({"y": [3]}) | |
| before = expand(pf.col("x")).alias("out") | |
| first, second = a.flat_map(expand), b.flat_map(expand) | |
| self.assertEqual(first.columns, ["f0"]) | |
| after = expand(pf.col("x")).alias("out") | |
| combined = first.union_all(second) | |
| for call in (before, after): | |
| lateral = a.join_lateral(call).select("out") | |
| combined = combined.union_all(lateral) | |
| self.assertCountEqual(combined.collect(), [Row(1), Row(2)] * 3 + [Row(3), Row(4)]) | |
| def test_table_function_and_callable_classes_and_instances(self): | |
| client_pid = os.getpid() | |
| class Expand(TableFunction): | |
| def __init__(self): | |
| self.constructor_pid = os.getpid() | |
| def open(self, context): | |
| self.offset = 10 | |
| def eval(self, row: Dict[str, Any]) -> Iterator[int]: | |
| if self.constructor_pid != client_pid: | |
| raise AssertionError("TableFunction must be constructed on the client") | |
| yield row["x"] + self.offset | |
| class Repeat: | |
| def __call__(self, row: Dict[str, Any]) -> Iterator[int]: | |
| yield row["x"] | |
| yield row["x"] | |
| source = pf.from_dict({"x": [1]}) | |
| first = source.flat_map(pf.udtf(Expand)) | |
| second = source.flat_map(pf.udtf(Expand())) | |
| combined = first.union_all(second) | |
| for func in (Repeat, Repeat(), pf.udtf(Repeat), pf.udtf(Repeat())): | |
| combined = combined.union_all(source.flat_map(func)) | |
| self.assertCountEqual(combined.collect(), [Row(11)] * 2 + [Row(1)] * 8) | |
| def test_wrapped_callable_class_methods(self): | |
| def transparent(func): | |
| @functools.wraps(func) | |
| def wrapper(*args, **kwargs): | |
| return func(*args, **kwargs) | |
| return wrapper | |
| class InstanceMethod: | |
| @transparent | |
| def __call__(self, row: Dict[str, Any]) -> Iterator[int]: | |
| yield row["x"] | |
| class StaticMethod: | |
| @staticmethod | |
| @transparent | |
| def __call__(row: Dict[str, Any]) -> Iterator[int]: | |
| yield row["x"] | |
| class ClassMethod: | |
| @classmethod | |
| @transparent | |
| def __call__(cls, row: Dict[str, Any]) -> Iterator[int]: | |
| yield row["x"] | |
| class InheritedStaticMethod(StaticMethod): | |
| pass | |
| def expand(row: Dict[str, Any]) -> Iterator[int]: | |
| yield row["x"] | |
| class WrappedFunction: | |
| @functools.wraps(expand) | |
| def __call__(self, *args, **kwargs): | |
| return expand(*args, **kwargs) | |
| source = pf.from_dict({"x": [1]}) | |
| combined = None | |
| for cls in (InstanceMethod, StaticMethod, ClassMethod, | |
| InheritedStaticMethod, WrappedFunction): | |
| self.assertEqual(list(cls()({"x": 1})), [1]) | |
| for func in (cls(), pf.udtf(cls())): | |
| self.assertEqual(source.flat_map(func).columns, ["f0"]) | |
| for func in (cls, pf.udtf(cls)): | |
| result = source.flat_map(func) | |
| combined = result if combined is None else combined.union_all(result) | |
| self.assertCountEqual(combined.collect(), [Row(1)] * 10) | |
| def test_nested_values_and_nulls(self): | |
| dtype = pf.DataType.struct({ | |
| "payload": pf.DataType.struct({"label": pf.DataType.string()}), | |
| "values": pf.DataType.list(pf.DataType.int64()), | |
| "attributes": pf.DataType.map(pf.DataType.string(), pf.DataType.int64()), | |
| "missing": pf.DataType.string(), | |
| }) | |
| def expand(row: Optional[Dict[str, Any]]): | |
| if row["x"] == 0: | |
| return None | |
| return {"payload": {"label": "ok"}, "values": [1, None], "attributes": {"a": 2}} | |
| result = pf.from_dict({"x": [0, 1]}).flat_map(expand, return_dtype=dtype) | |
| # Table.collect cannot decode NULL array elements, so inspect them in the JVM. | |
| projected = result.select( | |
| "payload", pf.col("values").cardinality, pf.col("values").at(1), | |
| pf.col("values").at(2).is_null, "attributes", "missing") | |
| self.assertEqual(projected.collect(), [Row(Row("ok"), 2, 1, True, {"a": 2}, None)]) | |
| def test_map_output_is_one_column(self): | |
| def expand(row) -> Iterator[Dict[str, int]]: | |
| yield {"value": row["x"]} | |
| yield {"value": row["x"] + 1} | |
| result = pf.from_dict({"x": [1]}).flat_map(expand) | |
| self.assertEqual(result.columns, ["f0"]) | |
| self.assertEqual(result.schema.get_field_data_types(), | |
| [DataTypes.MAP(DataTypes.STRING().not_null(), | |
| DataTypes.BIGINT().not_null()).not_null()]) | |
| self.assertCountEqual(result.collect(), [Row({"value": 1}), Row({"value": 2})]) | |
| class _DataFrameLateralJoinTests(_DataFrameUDTFTests): | |
| def test_lateral_join_semantics_and_predicates(self): | |
| @pf.udtf | |
| def words(text: Optional[str]) -> Iterator[str]: | |
| yield from (text or "").split() | |
| class Token(TypedDict): | |
| word: str | |
| length: int | |
| @pf.udtf | |
| def tokenize(text: Optional[str]) -> Iterator[Token]: | |
| for word in (text or "").split(): | |
| yield {"word": word, "length": len(word)} | |
| source = pf.from_records([(0, ""), (1, "a bb"), (2, "z"), (-1, "skip"), (3, None)], | |
| schema=["id", "text"]) | |
| call = words(pf.col("text")).alias("word") | |
| inner = source.join_lateral(call) | |
| left = source.join_lateral(call, ignore_empty=False, on=pf.lit(True)) | |
| filtered = source.join_lateral(tokenize(pf.col("text")), | |
| on=(pf.col("id") > 0) & (pf.col("length") > 1)) | |
| raw = table_udtf(lambda text: (text or "").split(), result_types=[DataTypes.STRING()]) | |
| legacy = source.join_lateral(raw(pf.col("text")).alias("word")) | |
| frames = [("inner", inner), ("left", left), ("filtered", filtered), ("raw", legacy)] | |
| combined = None | |
| for name, frame in frames: | |
| result = frame.select(pf.lit(name).alias("kind"), "id", "text", "word") | |
| combined = result if combined is None else combined.union_all(result) | |
| emitted = [(1, "a bb", "a"), (1, "a bb", "bb"), (2, "z", "z"), (-1, "skip", "skip")] | |
| expected = [Row(name, *row) for name in ("inner", "left", "raw") for row in emitted] | |
| expected += [Row("left", 0, "", None), Row("left", 3, None, None), | |
| Row("filtered", 1, "a bb", "bb")] | |
| self.assertCountEqual(combined.collect(), expected) | |
| class DataFrameUDTFStreamTests( | |
| _DataFrameFlatMapTests, _DataFrameLateralJoinTests, PyFlinkStreamDataFrameTestCase | |
| ): | |
| pass | |
| class DataFrameUDTFBatchTests( | |
| _DataFrameFlatMapTests, _DataFrameLateralJoinTests, PyFlinkBatchTableTestCase | |
| ): | |
| pass | |
| class DataFrameUDTFThreadTests( | |
| _DataFrameFlatMapTests, _DataFrameLateralJoinTests, PyFlinkStreamDataFrameTestCase | |
| ): | |
| def setUp(self): | |
| super().setUp() | |
| self.t_env.get_config().set("python.execution-mode", "thread") | |
| if __name__ == "__main__": | |
| unittest.main() |