blob: 86f51b655714ed1925f60135561717d3f184c813 [file]
################################################################################
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
################################################################################
import datetime
import functools
import os
import unittest
from typing import TypedDict
import cloudpickle
import pandas as pd
import pyflink.dataframe as pf
from pyflink.common import Row, Types
from pyflink.table import DataTypes, ListView, MapView, Schema
from pyflink.table.expressions import col, lit
from pyflink.table.udf import AggregateFunction, ScalarFunction, udaf as table_udaf
from pyflink.table.window import Session
from pyflink.testing.test_case_utils import (
PyFlinkBatchTableTestCase, PyFlinkDataFrameUTTestCase, PyFlinkStreamDataFrameTestCase,
)
from pyflink.util.exceptions import TableException
class Sum(AggregateFunction):
def create_accumulator(self):
return [0]
def accumulate(self, accumulator, value):
if value is not None:
accumulator[0] += value
def retract(self, accumulator, value):
if value is not None:
accumulator[0] -= value
def merge(self, accumulator, accumulators):
for other in accumulators:
accumulator[0] += other[0]
def get_value(self, accumulator) -> int:
return accumulator[0]
def get_accumulator_type(self):
return DataTypes.ARRAY(DataTypes.BIGINT())
class Statistics(TypedDict):
total: int
count: int
def _aggregate(declaration, *columns):
adapter = cloudpickle.loads(cloudpickle.dumps(declaration._table_udf_wrapper._func))
adapter.open(None)
try:
accumulator = adapter.create_accumulator()
adapter.accumulate(accumulator, *columns)
return adapter.get_value(accumulator)
finally:
adapter.close()
class DataFrameUDAFDeclarationTests(unittest.TestCase):
def test_declaration_forms(self):
for declaration in (pf.udaf(Sum), pf.udaf()(Sum), pf.udaf(return_dtype=int)(Sum),
pf.udaf(Sum())):
self.assertEqual(declaration.return_dtype, pf.DataType.int64().not_null())
self.assertEqual(declaration.__name__, "Sum")
def test_general_requires_aggregate_function(self):
class CallableSum:
def __call__(self, values) -> int:
return sum(values)
for func in (sum, CallableSum, CallableSum()):
with self.subTest(func=func), self.assertRaisesRegex(TypeError, "AggregateFunction"):
pf.udaf(func, return_dtype=int, accumulator_type=list[int])
def test_type_precedence(self):
class TypedSum(Sum):
def get_result_type(self):
return DataTypes.INT()
declaration = pf.udaf(TypedSum)
self.assertEqual(declaration.return_dtype, pf.DataType.int32())
self.assertEqual(declaration.accumulator_type, pf.DataType.list(pf.DataType.int64()))
declaration = pf.udaf(TypedSum, return_dtype=float, accumulator_type=list[float])
self.assertEqual(declaration.return_dtype, pf.DataType.float64().not_null())
self.assertEqual(declaration.accumulator_type,
pf.DataType.list(pf.DataType.float64().not_null()).not_null())
class ExplicitSum(Sum):
def get_result_type(self):
raise AssertionError("Explicit return_dtype must take precedence")
def get_accumulator_type(self):
raise AssertionError("Explicit accumulator_type must take precedence")
self.assertEqual(pf.udaf(ExplicitSum, return_dtype=int,
accumulator_type=list[int]).return_dtype,
pf.DataType.int64().not_null())
def test_return_annotation_is_resolved_independently(self):
def total(values):
return values.sum()
total.__annotations__ = {"values": "UnknownInput", "return": int}
self.assertEqual(pf.udaf(total, func_type="pandas").return_dtype,
pf.DataType.int64().not_null())
total.__annotations__["return"] = "UnknownOutput"
self.assertEqual(pf.udaf(total, return_dtype=int, func_type="pandas").return_dtype,
pf.DataType.int64().not_null())
def test_missing_and_invalid_types(self):
class UntypedSum(Sum):
get_accumulator_type = AggregateFunction.get_accumulator_type
def create_accumulator(self) -> list[int]:
raise AssertionError("Type resolution must not execute create_accumulator")
with self.assertRaisesRegex(TypeError, "accumulator_type"):
pf.udaf(UntypedSum)
with self.assertRaisesRegex(TypeError, "return_dtype"):
pf.udaf(lambda values: values.sum(), func_type="pandas")
with self.assertRaisesRegex(TypeError, "MAP"):
pf.udaf(lambda values: {}, return_dtype=dict[str, int], func_type="pandas")
with self.assertRaises(TypeError):
pf.udaf(Sum, return_dtype=object())
class InvalidType(Sum):
def get_result_type(self):
return int
with self.assertRaisesRegex(TypeError, "get_result_type"):
pf.udaf(InvalidType)
def test_invalid_functions_and_async_methods(self):
class Scalar(ScalarFunction):
def eval(self, value):
return value
class NeedsArgument(Sum):
def __init__(self, argument):
self.argument = argument
for func in (42, Scalar, Scalar(), NeedsArgument):
with self.subTest(func=func), self.assertRaises(TypeError):
pf.udaf(func, return_dtype=int, func_type="pandas")
with self.assertRaises(ValueError):
pf.udaf(Sum, func_type="arrow")
with self.assertRaises(TypeError):
pf.udaf(Sum, deterministic=1)
async def coroutine(values):
return 1
async def generator(values):
yield 1
@functools.wraps(generator)
def wrapped(values):
return generator(values)
for func in (coroutine, generator, wrapped):
with self.subTest(func=func), self.assertRaisesRegex(TypeError, "async"):
pf.udaf(func, return_dtype=int, func_type="pandas")
for method in ("accumulate", "get_value", "create_accumulator", "open", "merge"):
cls = type("AsyncAggregate", (Sum,), {method: coroutine})
with self.subTest(method=method), self.assertRaisesRegex(TypeError, "async"):
pf.udaf(cls)
def test_names_and_determinism(self):
for name in (None, "", "my_sum"):
self.assertEqual(pf.udaf(Sum, name=name).__name__, name or "Sum")
class RandomSum(Sum):
def is_deterministic(self):
return False
with self.assertRaisesRegex(ValueError, "Inconsistent deterministic"):
pf.udaf(RandomSum)
adapter = pf.udaf(RandomSum, deterministic=False)._table_udf_wrapper._func
self.assertFalse(adapter.is_deterministic())
def test_client_constructor_and_worker_lifecycle(self):
events = []
class LifecycleSum(Sum):
def __init__(self):
events.append("init")
def open(self, context):
events.append(("open", context))
def create_accumulator(self):
events.append("accumulator")
return super().create_accumulator()
def close(self):
events.append("close")
raise RuntimeError("close failure")
declaration = pf.udaf(LifecycleSum)
adapter = declaration._table_udf_wrapper._func
self.assertEqual(events, ["init"])
adapter.open("worker")
accumulator = adapter.create_accumulator()
adapter.accumulate(accumulator, 5)
adapter.retract(accumulator, 2)
adapter.merge(accumulator, [[7], [1]])
self.assertEqual(accumulator, [11])
self.assertEqual(adapter.get_value(accumulator), 11)
with self.assertRaisesRegex(RuntimeError, "close failure"):
adapter.close()
self.assertEqual(events, ["init", ("open", "worker"), "accumulator", "close"])
with self.assertRaisesRegex(RuntimeError, "before open"):
adapter.get_value(accumulator)
adapter.close()
def test_callable_classes_construct_on_worker(self):
events = []
class Total:
def __init__(self, offset=0):
events.append("init")
self.offset = offset
def __call__(self, values) -> int:
return int(values.sum()) + self.offset
declaration = pf.udaf(Total, func_type="pandas")
adapter = declaration._table_udf_wrapper._func
self.assertEqual(events, [])
adapter.open(None)
try:
acc = adapter.create_accumulator()
adapter.accumulate(acc, pd.Series([1, 2]))
self.assertEqual(adapter.get_value(acc), 3)
finally:
adapter.close()
self.assertEqual(events, ["init"])
for func in (Total(2), functools.partial(lambda offset, values: values.sum() + offset, 2)):
self.assertEqual(_aggregate(pf.udaf(func, return_dtype=int, func_type="pandas"),
pd.Series([1, 2])), 5)
def test_struct_normalization_and_user_failures(self):
dtype = pf.DataType.struct({
"nested": pf.DataType.list(pf.DataType.struct({"value": pf.DataType.int64()})),
"missing": pf.DataType.string(),
})
class Nested(Sum):
def get_value(self, accumulator):
return {"nested": [{"value": accumulator[0]}]}
for func_type, expected in (
("general", Row(nested=[Row(value=3)], missing=None)),
("pandas", ([(3,)], None)),
):
with self.subTest(func_type=func_type):
self.assertEqual(_aggregate(pf.udaf(
Nested, return_dtype=dtype, func_type=func_type), 3), expected)
def fail(values) -> int:
raise RuntimeError("user failure")
with self.assertRaisesRegex(RuntimeError, "user failure"):
_aggregate(pf.udaf(fail, func_type="pandas"), pd.Series([1]))
class DataFrameUDAFPlanningTests(PyFlinkDataFrameUTTestCase):
def test_sql_type_declarations(self):
class SqlSum(Sum):
def get_result_type(self):
return "BIGINT"
def get_accumulator_type(self):
return "ARRAY<BIGINT>"
for declaration in (pf.udaf(SqlSum), pf.udaf(
Sum, return_dtype="BIGINT", accumulator_type="ARRAY<BIGINT>")):
self.assertEqual(declaration.return_dtype, pf.DataType.int64())
self.assertEqual(declaration.accumulator_type, pf.DataType.list(pf.DataType.int64()))
class DataFrameUDAFBatchTests(PyFlinkBatchTableTestCase):
def setUp(self):
previous = pf.get_table_environment()
self.addCleanup(pf.set_table_environment, previous)
pf.set_table_environment(self.t_env)
def test_pandas_callables_classes_and_multiple_columns(self):
client_pid = os.getpid()
class Total:
def __init__(self):
if os.getpid() == client_pid:
raise AssertionError("Callable class must be constructed on the worker")
def __call__(self, values, weights) -> int:
return int((values * weights).sum())
class WeightedSum(Sum):
def __init__(self):
self.pid = os.getpid()
def open(self, context):
if self.pid != client_pid:
raise AssertionError("AggregateFunction must be constructed on the client")
self.opened = True
def accumulate(self, accumulator, values, weights):
if not self.opened:
raise AssertionError("open must be called")
accumulator[0] += int((values * weights).sum())
source = pf.from_records([("a", 1, 2), ("a", 3, 4), ("b", 2, 3)],
schema=["category", "value", "weight"])
weighted = pf.udaf(Total, func_type="pandas")
result = source.group_by("category").agg(
total=weighted(pf.col("value"), pf.col("weight")))
self.assertEqual(result.columns, ["category", "total"])
self.assertEqual(result.schema.get_field_data_types(),
[DataTypes.STRING(), DataTypes.BIGINT().not_null()])
second = source.group_by("category").agg(
total=pf.udaf(WeightedSum, func_type="pandas")(pf.col("value"), pf.col("weight")))
self.assertCountEqual(result.union_all(second).collect(),
[Row("a", 14), Row("b", 6)] * 2)
def test_sql_bindings_and_struct_fields(self):
@pf.udaf(func_type="pandas")
def stats(values) -> Statistics:
return {"count": len(values), "total": int(values.sum())}
source = pf.from_dict({"value": [1, 2, 3]})
direct = source.agg(stats(pf.col("value")).alias("summary"))
explicit = pf.sql("SELECT s(`value`) AS summary FROM src", auto_bind=False,
src=source, s=stats)
automatic = pf.sql("SELECT stats(`value`) AS summary FROM source")
for name in ("stats", "s"):
self.assertNotIn(name, self.t_env.list_user_defined_functions())
combined = direct.union_all(explicit).union_all(automatic)
projected = combined.select(pf.col("summary").get("total"),
pf.col("summary").get("count"))
self.assertCountEqual(projected.collect(), [Row(6, 3)] * 3)
def test_pandas_nested_struct_results(self):
def summarize(values):
return {"nested": {
"sum": {"label": "total", "value": int(values.sum())},
}}
class Summary(AggregateFunction):
def create_accumulator(self):
return [None]
def accumulate(self, accumulator, values):
accumulator[0] = summarize(values)
def get_value(self, accumulator):
return accumulator[0]
dtype = "ROW<nested MAP<STRING, ROW<value BIGINT, label STRING>>>"
function = pf.udaf(summarize, return_dtype=dtype, func_type="pandas")
aggregate = pf.udaf(Summary, return_dtype=dtype, func_type="pandas")
source = pf.from_dict({"value": [1, 2, 3]})
result = source.agg(function(pf.col("value")), aggregate(pf.col("value")))
expected = Row({"sum": Row(6, "total")})
self.assertEqual(result.collect(), [Row(expected, expected)])
def test_empty_nulls_and_table_udaf_compatibility(self):
def total(values) -> int:
return int(values.sum())
source = pf.from_dict({"value": [1, None, 3]})
declaration = pf.udaf(total, func_type="pandas")
native = table_udaf(total, result_type=DataTypes.BIGINT(), func_type="pandas")
self.assertEqual(source.agg(total=declaration(pf.col("value"))).collect(), [Row(4)])
empty = source.filter(pf.col("value") < 0)
self.assertEqual(empty.agg(total=declaration(pf.col("value"))).collect(),
empty.agg(total=native(pf.col("value"))).collect())
with self.assertRaisesRegex(TableException, "non-Pandas UDAFs"):
source.agg(total=pf.udaf(Sum)(pf.col("value"))).to_table().explain()
class DataFrameUDAFStreamTests(PyFlinkStreamDataFrameTestCase):
def test_grouped_global_and_retracting_inputs(self):
total = pf.udaf(Sum)
source = pf.from_records([("a", 1), ("a", 2), ("b", 3), ("b", None)],
schema=["category", "value"])
grouped = source.group_by("category").agg(total=total(pf.col("value")))
self.assertEqual(self._materialize(grouped, key=["category"]), [("a", 3), ("b", 3)])
self.assertEqual(self._materialize(source.agg(total=total(pf.col("value"))), key=[]),
[(6,)])
intermediate = source.group_by("category").agg(subtotal=pf.col("value").sum)
result = intermediate.agg(total=total(pf.col("subtotal")))
self.assertEqual(self._materialize(result, key=[]), [(6,)])
with self.assertRaisesRegex(TableException, "Pandas UDAFs"):
source.agg(total=pf.udaf(lambda values: values.sum(), return_dtype=int,
func_type="pandas")(pf.col("value"))).to_table().explain()
def test_general_sql_and_empty_input(self):
total = pf.udaf(Sum())
source = pf.from_dict({"value": [1, 2, 3]})
result = pf.sql("SELECT total(`value`) AS total_value FROM source",
auto_bind=False, source=source, total=total)
self.assertEqual(self._materialize(result, key=[]), [(6,)])
empty = source.filter(pf.col("value") < 0)
native = table_udaf(Sum(), result_type=DataTypes.BIGINT())
self.assertEqual(self._materialize(empty.agg(total(pf.col("value"))), key=[]),
self._materialize(empty.agg(native(pf.col("value"))), key=[]))
def test_dataview_accumulator_is_preserved(self):
class DistinctCount(AggregateFunction):
def create_accumulator(self):
return Row(MapView(), ListView())
def accumulate(self, accumulator, value):
if value not in accumulator[0]:
accumulator[0][value] = True
accumulator[1].add(value)
def get_value(self, accumulator) -> int:
return len(list(accumulator[1].get()))
def get_accumulator_type(self):
return DataTypes.ROW([
DataTypes.FIELD("seen", DataTypes.MAP_VIEW(DataTypes.STRING(),
DataTypes.BOOLEAN())),
DataTypes.FIELD("values", DataTypes.LIST_VIEW(DataTypes.STRING())),
])
source = pf.from_records([("a", "x"), ("a", "x"), ("a", "y"), ("b", "x")],
schema=["category", "value"])
result = source.group_by("category").agg(count=pf.udaf(DistinctCount)(pf.col("value")))
self.assertEqual(self._materialize(result, key=["category"]), [("a", 2), ("b", 1)])
def test_session_window_merges_accumulators(self):
data = self.env.from_collection([
Row("a", 1, datetime.datetime(2020, 1, 1, 0, 0, 1)),
Row("a", 3, datetime.datetime(2020, 1, 1, 0, 0, 3)),
Row("a", 2, datetime.datetime(2020, 1, 1, 0, 0, 2)),
], type_info=Types.ROW_NAMED(["category", "value", "ts"],
[Types.STRING(), Types.INT(), Types.SQL_TIMESTAMP()]))
schema = Schema.new_builder().column("category", DataTypes.STRING()) \
.column("value", DataTypes.INT()) \
.column("ts", DataTypes.TIMESTAMP(3).bridged_to("java.sql.Timestamp")) \
.watermark("ts", "ts - INTERVAL '10' SECOND").build()
table = self.t_env.from_data_stream(data, schema)
total = pf.udaf(Sum)
result = table.window(Session.with_gap(lit(2).seconds).on(col("ts")).alias("w")) \
.group_by(col("category"), col("w")) \
.select(col("category"), total(col("value")).alias("total"))
self.assertEqual(pf.from_table(result).collect(), [Row("a", 6)])
if __name__ == "__main__":
unittest.main()