blob: f5301e9e74b1258ad7cfe974cbff909b9a468186 [file]
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Handlers for the pandas UDF eval types (the UDF exchanges ``pd.Series`` /
``pd.DataFrame`` values, converted to and from Arrow via the conversion utilities
in ``pyspark.sql.conversion``, the pandas counterpart of ``ArrowBatchTransformer``).
pandas and pyarrow are imported lazily (inside ``__init__``/``run`` and the type-checking
block) so the module stays importable and its handlers register without either installed.
Each handler calls ``require_minimum_pandas_version`` and
``require_minimum_pyarrow_version`` in ``__init__`` so a missing or too-old
dependency surfaces a clear error when the handler runs.
"""
from __future__ import annotations
import inspect
from collections.abc import Iterator
from typing import TYPE_CHECKING, Any
from pyspark import _NoValue
from pyspark.errors import PySparkRuntimeError, PySparkTypeError, PySparkValueError
from pyspark.eval_handlers._base import (
BatchEvalTypeHandler,
CoGroupedEvalTypeHandler,
GroupedEvalTypeHandler,
)
from pyspark.eval_handlers.utils import extract_key_value_indexes, wrap_kwargs_support
from pyspark.eval_handlers.verification import (
verify_iter_result_row_count,
verify_iterator_exhausted,
verify_output_row_limit,
verify_pandas_result,
verify_result_row_count,
verify_return_type,
)
from pyspark.sql.conversion import (
ArrowBatchTransformer,
ArrowToPandasConversion,
PandasToArrowConversion,
)
from pyspark.sql.pandas.utils import (
require_minimum_pandas_version,
require_minimum_pyarrow_version,
)
from pyspark.sql.types import BinaryType, StringType, StructField, StructType
from pyspark.util import PythonEvalType, fail_on_stopiteration
if TYPE_CHECKING:
import pyarrow as pa
from pyspark.eval_handlers._typing import UDF, CoGroupedBatch, GroupedBatch
from pyspark.worker_util import EvalConf, RunnerConf
class PandasScalarUDFHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
"""SQL_SCALAR_PANDAS_UDF: convert each input RecordBatch to pandas Series
(struct columns become DataFrames), invoke each UDF once per batch, check the
row count, and convert the pandas results back to one RecordBatch. The pandas
counterpart of ArrowScalarUDFHandler, calling the Arrow<->pandas conversions in
``pyspark.sql.conversion`` directly with the runner_conf-derived parameters."""
eval_type = PythonEvalType.SQL_SCALAR_PANDAS_UDF
_multi_udf = True
def __init__(
self, udf_descs: list[UDF], runner_conf: "RunnerConf", eval_conf: "EvalConf"
) -> None:
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
self._return_schema = StructType(
[StructField("_%d" % i, udf.return_type) for i, udf in enumerate(udf_descs)]
)
def run(self, split_index: int, data: "Iterator[pa.RecordBatch]") -> "Iterator[pa.RecordBatch]":
import pandas as pd
for input_batch in data:
num_rows = input_batch.num_rows
# Input: Arrow -> pandas Series (struct columns become DataFrames).
pandas_columns = ArrowToPandasConversion.to_pandas(
input_batch,
timezone=self._runner_conf.timezone,
struct_in_pandas="dict",
ndarray_as_list=False,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
df_for_struct=True,
)
# Process: evaluate each UDF column-wise on pandas Series.
results = []
for udf, func in zip(self._udf_descs, self._funcs):
return_type = udf.return_type
result = func(
*[pandas_columns[o] for o in udf.args_offsets],
**{k: pandas_columns[v] for k, v in udf.kwargs_offsets.items()},
)
if not hasattr(result, "__len__"):
pd_type = (
"pandas.DataFrame"
if isinstance(return_type, StructType)
else "pandas.Series"
)
raise PySparkTypeError(
errorClass="UDF_RETURN_TYPE",
messageParameters={"expected": pd_type, "actual": type(result).__name__},
)
verify_result_row_count(len(result), num_rows)
# struct_in_pandas="dict": UDF must return a DataFrame for struct types.
if isinstance(return_type, StructType) and not isinstance(result, pd.DataFrame):
raise PySparkValueError(
"Invalid return type. Please make sure that the UDF returns a "
"pandas.DataFrame when the specified return type is StructType."
)
results.append(result)
# Output: pandas -> Arrow.
yield PandasToArrowConversion.from_pandas(
results,
self._return_schema,
timezone=self._runner_conf.timezone,
safecheck=self._runner_conf.safecheck,
arrow_cast=True,
prefers_large_types=self._runner_conf.use_large_var_types,
assign_cols_by_name=self._runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=self._runner_conf.int_to_decimal_coercion_enabled,
)
class PandasScalarIterUDFHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
"""SQL_SCALAR_PANDAS_ITER_UDF: the single UDF receives an iterator of its
argument columns (each a pandas Series, struct columns as DataFrames) and yields
an iterator of pandas Series/DataFrames; enforce the declared element type, verify
the total row count matches the input, and convert each result back to one
RecordBatch. The pandas counterpart of ArrowScalarIterUDFHandler, calling the
Arrow<->pandas conversions in ``pyspark.sql.conversion`` directly."""
eval_type = PythonEvalType.SQL_SCALAR_PANDAS_ITER_UDF
_iter_based = True
def __init__(
self, udf_descs: list[UDF], runner_conf: "RunnerConf", eval_conf: "EvalConf"
) -> None:
import pandas as pd
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
self._return_schema = StructType([StructField("_0", self._udf_desc.return_type)])
self._expected_iter_type = (
Iterator[pd.DataFrame]
if isinstance(self._udf_desc.return_type, StructType)
else Iterator[pd.Series]
)
def run(self, split_index: int, data: "Iterator[pa.RecordBatch]") -> "Iterator[pa.RecordBatch]":
num_input_rows = 0
def extract_args(batch: "pa.RecordBatch") -> Any:
nonlocal num_input_rows
# Input: Arrow -> pandas Series (struct columns become DataFrames).
pandas_columns = ArrowToPandasConversion.to_pandas(
batch,
timezone=self._runner_conf.timezone,
struct_in_pandas="dict",
ndarray_as_list=False,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
df_for_struct=True,
)
args = tuple(pandas_columns[o] for o in self._udf_desc.args_offsets)
num_input_rows += batch.num_rows
return args[0] if len(args) == 1 else args
# Extract args from input batches (streaming), then call the UDF and verify the
# result type (iterator of pd.Series / pd.DataFrame).
args_iter = map(extract_args, data)
verified_iter = verify_return_type(self._func(args_iter), self._expected_iter_type)
def convert(result: Any) -> "pa.RecordBatch":
verify_pandas_result(
result,
self._udf_desc.return_type,
assign_cols_by_name=True,
truncate_return_schema=True,
)
return PandasToArrowConversion.from_pandas(
[result],
self._return_schema,
timezone=self._runner_conf.timezone,
safecheck=self._runner_conf.safecheck,
arrow_cast=True,
prefers_large_types=self._runner_conf.use_large_var_types,
assign_cols_by_name=self._runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=self._runner_conf.int_to_decimal_coercion_enabled,
)
# Row-limit check (fail-fast) then exact row-count match (final).
batches = verify_output_row_limit(map(convert, verified_iter), lambda: num_input_rows)
yield from verify_iter_result_row_count(batches, lambda: num_input_rows)
# Verify the input iterator was fully consumed.
verify_iterator_exhausted(args_iter)
class PandasMapUDFHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
"""SQL_MAP_PANDAS_ITER_UDF (mapInPandas): the single UDF receives the input
RecordBatch stream as an iterator of pandas DataFrames (one per batch, the single
struct column expanded) and yields pandas DataFrames/Series, converted back to one
RecordBatch each. The pandas counterpart of ArrowMapUDFHandler."""
eval_type = PythonEvalType.SQL_MAP_PANDAS_ITER_UDF
_iter_based = True
def __init__(
self, udf_descs: list[UDF], runner_conf: "RunnerConf", eval_conf: "EvalConf"
) -> None:
import pandas as pd
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
self._return_schema = StructType([StructField("_0", self._udf_desc.return_type)])
self._expected_iter_type = (
Iterator[pd.DataFrame]
if isinstance(self._udf_desc.return_type, StructType)
else Iterator[pd.Series]
)
def run(self, split_index: int, data: "Iterator[pa.RecordBatch]") -> "Iterator[pa.RecordBatch]":
def dataframe_iter() -> "Iterator[Any]":
# Input batches have a single struct column (see MapInBatchEvaluatorFactory);
# convert lazily so peakmem stays bounded by one batch.
for batch in data:
yield ArrowToPandasConversion.to_pandas(
batch,
timezone=self._runner_conf.timezone,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
df_for_struct=True,
)[0]
result = self._func(dataframe_iter())
# The declared signature is Iterator[...], so a strict iterator is required by
# default. With the legacy flag, accept any object Python can iterate over -- via
# iter(...), which honors both __iter__ and the sequence protocol (__getitem__) --
# by adapting it into an iterator before the shared element-type verification.
if self._runner_conf.map_in_batch_legacy_accept_any_iterable and not isinstance(
result, Iterator
):
try:
result = iter(result)
except TypeError:
# Not iterable at all; leave it so verify_return_type below raises the
# standard UDF_RETURN_TYPE error.
pass
for df in verify_return_type(result, self._expected_iter_type):
verify_pandas_result(
df,
self._udf_desc.return_type,
assign_cols_by_name=True,
truncate_return_schema=True,
)
yield PandasToArrowConversion.from_pandas(
[df],
self._return_schema,
timezone=self._runner_conf.timezone,
safecheck=self._runner_conf.safecheck,
arrow_cast=True,
prefers_large_types=self._runner_conf.use_large_var_types,
assign_cols_by_name=self._runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=self._runner_conf.int_to_decimal_coercion_enabled,
)
class PandasCoGroupedMapUDFHandler(CoGroupedEvalTypeHandler["pa.RecordBatch"]):
"""SQL_COGROUPED_MAP_PANDAS_UDF (applyInPandas on a cogroup): the single UDF
receives the two sides' value columns as pandas DataFrames (plus the grouping
key when it takes three arguments) and returns one pandas DataFrame, converted
back to one RecordBatch. The pandas counterpart of ArrowCoGroupedMapUDFHandler."""
eval_type = PythonEvalType.SQL_COGROUPED_MAP_PANDAS_UDF
def __init__(
self, udf_descs: list[UDF], runner_conf: "RunnerConf", eval_conf: "EvalConf"
) -> None:
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
# signature was lost when wrapping it
self._num_udf_args = len(inspect.getfullargspec(self._raw_func).args)
parsed_offsets = extract_key_value_indexes(self._udf_desc.args_offsets)
self._left_key_offsets, self._left_value_offsets = parsed_offsets[0]
self._right_key_offsets, self._right_value_offsets = parsed_offsets[1]
self._return_schema = StructType([StructField("_0", self._udf_desc.return_type)])
def run(self, split_index: int, data: "Iterator[CoGroupedBatch]") -> "Iterator[pa.RecordBatch]":
"""Apply cogroupBy Pandas UDF.
The explicit ``del`` calls keep peakmem bounded across groups: without them the
previous group's tables and DataFrames stay bound on the generator frame while
the next group is read.
"""
import pandas as pd
import pyarrow as pa
for left_batches, right_batches in data:
left_table = pa.Table.from_batches(left_batches)
right_table = pa.Table.from_batches(right_batches)
left_series = ArrowToPandasConversion.to_pandas(
left_table,
timezone=self._runner_conf.timezone,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
)
right_series = ArrowToPandasConversion.to_pandas(
right_table,
timezone=self._runner_conf.timezone,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
)
left_df = pd.concat([left_series[o] for o in self._left_value_offsets], axis=1)
right_df = pd.concat([right_series[o] for o in self._right_value_offsets], axis=1)
if self._num_udf_args == 2:
result = self._func(left_df, right_df)
else:
key_series = (
[left_series[o] for o in self._left_key_offsets]
if not left_df.empty
else [right_series[o] for o in self._right_key_offsets]
)
key = tuple(s.iloc[0] for s in key_series)
result = self._func(key, left_df, right_df)
del left_batches, right_batches, left_table, right_table
del left_series, right_series, left_df, right_df
verify_pandas_result(
result,
self._udf_desc.return_type,
self._runner_conf.assign_cols_by_name,
truncate_return_schema=False,
)
yield PandasToArrowConversion.from_pandas(
[result],
self._return_schema,
timezone=self._runner_conf.timezone,
safecheck=self._runner_conf.safecheck,
arrow_cast=True,
prefers_large_types=self._runner_conf.use_large_var_types,
assign_cols_by_name=self._runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=self._runner_conf.int_to_decimal_coercion_enabled,
)
del result
class PandasGroupedMapIterUDFHandler(GroupedEvalTypeHandler["pa.RecordBatch"]):
"""SQL_GROUPED_MAP_PANDAS_ITER_UDF: the single UDF receives each group as an
iterator of pandas DataFrames (one per input batch, converted lazily) and returns
an iterator of pandas DataFrames, each converted back to one RecordBatch. The
pandas counterpart of ArrowGroupedMapIterUDFHandler."""
eval_type = PythonEvalType.SQL_GROUPED_MAP_PANDAS_ITER_UDF
_iter_based = True
def __init__(
self, udf_descs: list[UDF], runner_conf: "RunnerConf", eval_conf: "EvalConf"
) -> None:
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
# signature was lost when wrapping it
self._num_udf_args = len(inspect.getfullargspec(self._raw_func).args)
parsed_offsets = extract_key_value_indexes(self._udf_desc.args_offsets)
assert len(parsed_offsets) == 1, (
"Expected one pair of offsets for GROUPED_MAP_PANDAS_ITER UDF."
)
self._key_offsets, self._value_offsets = parsed_offsets[0]
self._return_schema = StructType([StructField("_0", self._udf_desc.return_type)])
def run(self, split_index: int, data: "Iterator[GroupedBatch]") -> "Iterator[pa.RecordBatch]":
"""Apply groupBy Pandas UDF (iterator variant)."""
import pandas as pd
value_offsets = self._value_offsets
for group in data:
group_iter = iter(group)
# Read the first batch to extract grouping keys.
first_series = ArrowToPandasConversion.to_pandas(
next(group_iter),
timezone=self._runner_conf.timezone,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
)
def dataframe_iter() -> "Iterator[pd.DataFrame]":
# Convert lazily so peakmem stays bounded by one batch, not the whole group.
yield pd.concat([first_series[o] for o in value_offsets], axis=1)
for batch in group_iter:
series = ArrowToPandasConversion.to_pandas(
batch,
timezone=self._runner_conf.timezone,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
)
yield pd.concat([series[o] for o in value_offsets], axis=1)
if self._num_udf_args == 1:
result = self._func(dataframe_iter())
else:
key = tuple(first_series[o].iloc[0] for o in self._key_offsets)
result = self._func(key, dataframe_iter())
for df in result:
verify_pandas_result(
df,
self._udf_desc.return_type,
self._runner_conf.assign_cols_by_name,
truncate_return_schema=False,
)
yield PandasToArrowConversion.from_pandas(
[df],
self._return_schema,
timezone=self._runner_conf.timezone,
safecheck=self._runner_conf.safecheck,
arrow_cast=True,
prefers_large_types=self._runner_conf.use_large_var_types,
assign_cols_by_name=self._runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=(
self._runner_conf.int_to_decimal_coercion_enabled
),
)
# Drain remaining input batches to maintain stream position.
for _ in group_iter:
pass
class PandasGroupedMapUDFHandler(GroupedEvalTypeHandler["pa.RecordBatch"]):
"""SQL_GROUPED_MAP_PANDAS_UDF (applyInPandas): the single UDF receives each group
as one pandas DataFrame of its value columns (plus the grouping key when it takes
two arguments) and returns one pandas DataFrame, converted back to Arrow and
resized toward the output batch byte cap. The pandas counterpart of
ArrowGroupedMapUDFHandler."""
eval_type = PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF
def __init__(
self, udf_descs: list[UDF], runner_conf: "RunnerConf", eval_conf: "EvalConf"
) -> None:
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
# signature was lost when wrapping it
self._num_udf_args = len(inspect.getfullargspec(self._raw_func).args)
parsed_offsets = extract_key_value_indexes(self._udf_desc.args_offsets)
assert len(parsed_offsets) == 1, "Expected one pair of offsets for GROUPED_MAP_PANDAS UDF."
self._key_offsets, self._value_offsets = parsed_offsets[0]
self._return_schema = StructType([StructField("_0", self._udf_desc.return_type)])
def run(self, split_index: int, data: "Iterator[GroupedBatch]") -> "Iterator[pa.RecordBatch]":
# Bound each output RecordBatch toward the byte cap (no limit when unset).
max_output_bytes = self._runner_conf.python_udf_arrow_worker_output_batch_max_bytes
if max_output_bytes > 0:
return ArrowBatchTransformer.resize_batches(self._run(data), max_output_bytes)
return self._run(data)
def _run(self, data: "Iterator[GroupedBatch]") -> "Iterator[pa.RecordBatch]":
"""Apply groupBy Pandas UDF (non-iterator variant).
The explicit ``del`` calls below keep peakmem bounded across groups. Without
them, generator locals from the previous iteration stay bound on the frame until
each statement in the next iteration rebinds its slot, so the input-side
DataFrames overlap with the next group's allocations and the working set grows
unbounded on wide-column, large-group inputs. ``del result`` runs on resume from
yield, before ``data.__next__()`` is asked for the next group.
"""
import pandas as pd
import pyarrow as pa
for group in data:
all_batches = list(group)
if all_batches:
table = pa.Table.from_batches(all_batches).combine_chunks()
else:
table = pa.table({})
all_series = ArrowToPandasConversion.to_pandas(
table,
timezone=self._runner_conf.timezone,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
)
value_df = pd.concat([all_series[o] for o in self._value_offsets], axis=1)
if self._num_udf_args == 1:
result = self._func(value_df)
else:
key = tuple(all_series[o].iloc[0] for o in self._key_offsets)
result = self._func(key, value_df)
del all_batches, table, all_series, value_df
verify_pandas_result(
result,
self._udf_desc.return_type,
self._runner_conf.assign_cols_by_name,
truncate_return_schema=False,
)
yield PandasToArrowConversion.from_pandas(
[result],
self._return_schema,
timezone=self._runner_conf.timezone,
safecheck=self._runner_conf.safecheck,
arrow_cast=True,
prefers_large_types=self._runner_conf.use_large_var_types,
assign_cols_by_name=self._runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=self._runner_conf.int_to_decimal_coercion_enabled,
)
del result
class PandasGroupedAggUDFHandler(GroupedEvalTypeHandler["pa.RecordBatch"]):
"""SQL_GROUPED_AGG_PANDAS_UDF: each UDF reduces its input columns (pandas Series)
over the whole group to a single scalar; emit one row per group with one column
per UDF, converted back to Arrow. The pandas counterpart of
ArrowGroupedAggUDFHandler."""
eval_type = PythonEvalType.SQL_GROUPED_AGG_PANDAS_UDF
_multi_udf = True
def __init__(
self, udf_descs: list[UDF], runner_conf: "RunnerConf", eval_conf: "EvalConf"
) -> None:
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
self._return_schema = StructType(
[StructField("_%d" % i, udf.return_type) for i, udf in enumerate(udf_descs)]
)
def run(self, split_index: int, data: "Iterator[GroupedBatch]") -> "Iterator[pa.RecordBatch]":
import pandas as pd
import pyarrow as pa
for group in data:
batch_list = list(group)
if not batch_list:
continue
table = pa.Table.from_batches(batch_list).combine_chunks()
all_series = ArrowToPandasConversion.to_pandas(
table,
timezone=self._runner_conf.timezone,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
)
results = [
func(
*[all_series[o] for o in udf.args_offsets],
**{k: all_series[v] for k, v in udf.kwargs_offsets.items()},
)
for udf, func in zip(self._udf_descs, self._funcs)
]
yield PandasToArrowConversion.from_pandas(
[pd.Series([r]) for r in results],
self._return_schema,
timezone=self._runner_conf.timezone,
safecheck=self._runner_conf.safecheck,
arrow_cast=True,
prefers_large_types=self._runner_conf.use_large_var_types,
assign_cols_by_name=self._runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=self._runner_conf.int_to_decimal_coercion_enabled,
)
class PandasGroupedAggIterUDFHandler(GroupedEvalTypeHandler["pa.RecordBatch"]):
"""SQL_GROUPED_AGG_PANDAS_ITER_UDF: the single UDF receives each group as an
iterator of its input columns (one pandas Series, or a tuple of Series, per input
batch) and returns one scalar; emit one row per group. The pandas counterpart of
ArrowGroupedAggIterUDFHandler."""
eval_type = PythonEvalType.SQL_GROUPED_AGG_PANDAS_ITER_UDF
def __init__(
self, udf_descs: list[UDF], runner_conf: "RunnerConf", eval_conf: "EvalConf"
) -> None:
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
self._return_schema = StructType([StructField("_0", self._udf_desc.return_type)])
def run(self, split_index: int, data: "Iterator[GroupedBatch]") -> "Iterator[pa.RecordBatch]":
import pandas as pd
args_offsets = self._udf_desc.args_offsets
def extract_series(batch: "pa.RecordBatch") -> Any:
# Convert one RecordBatch to a pandas Series per column, then select args:
# - pd.Series for a single column
# - tuple[pd.Series, ...] for multiple columns
all_series = ArrowToPandasConversion.to_pandas(
batch,
timezone=self._runner_conf.timezone,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
)
series = tuple(all_series[o] for o in args_offsets)
return series[0] if len(series) == 1 else series
for group in data:
series_iter = map(extract_series, group)
result = self._func(series_iter)
# Drain remaining batches to maintain stream position
for _ in series_iter:
pass
yield PandasToArrowConversion.from_pandas(
[pd.Series([result])],
self._return_schema,
timezone=self._runner_conf.timezone,
safecheck=self._runner_conf.safecheck,
arrow_cast=True,
prefers_large_types=False,
assign_cols_by_name=self._runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=self._runner_conf.int_to_decimal_coercion_enabled,
)
class PandasWindowAggUDFHandler(GroupedEvalTypeHandler["pa.RecordBatch"]):
"""SQL_WINDOW_AGG_PANDAS_UDF: each UDF produces one value per input row over its
window frame -- an unbounded frame is computed once and repeated for every row,
a bounded frame slices the input Series per row -- emitted as one column per UDF,
converted back to Arrow. The pandas counterpart of ArrowWindowAggUDFHandler."""
eval_type = PythonEvalType.SQL_WINDOW_AGG_PANDAS_UDF
_multi_udf = True
def __init__(
self, udf_descs: list[UDF], runner_conf: "RunnerConf", eval_conf: "EvalConf"
) -> None:
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
self._window_bound_types = runner_conf.window_bound_types
self._return_schema = StructType(
[StructField("_%d" % i, udf.return_type) for i, udf in enumerate(udf_descs)]
)
def run(self, split_index: int, data: "Iterator[GroupedBatch]") -> "Iterator[pa.RecordBatch]":
import pandas as pd
import pyarrow as pa
for group in data:
batch_list = list(group)
if not batch_list:
continue
table = pa.Table.from_batches(batch_list).combine_chunks()
all_series = ArrowToPandasConversion.to_pandas(
table,
timezone=self._runner_conf.timezone,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
)
num_rows = table.num_rows
result_series = []
for udf_index, (udf, udf_func) in enumerate(zip(self._udf_descs, self._funcs)):
args_offsets, kwargs_offsets = udf.args_offsets, udf.kwargs_offsets
bound_type = self._window_bound_types[udf_index]
if bound_type == "unbounded":
result = udf_func(
*[all_series[o] for o in args_offsets],
**{k: all_series[v] for k, v in kwargs_offsets.items()},
)
# Repeat the scalar result to match the window (group) length.
result_series.append(pd.Series([result]).repeat(num_rows))
elif bound_type == "bounded":
# args_offsets[0] and args_offsets[1] are begin_index and end_index.
assert len(args_offsets) >= 2, len(args_offsets)
# Index operation is faster on np.ndarray, so we turn the
# index series into np arrays here for performance.
begin_array = all_series[args_offsets[0]].values
end_array = all_series[args_offsets[1]].values
series = [all_series[o] for o in args_offsets[2:]]
kw_series = {k: all_series[v] for k, v in kwargs_offsets.items()}
results = []
for i in range(num_rows):
# Note: Creating a slice from a series for each window is
# actually pretty expensive. However, there
# is no easy way to reduce cost here.
# Note: s.iloc[i : j] is about 30% faster than s[i: j], with
# the caveat that the created slices shares the same
# memory with s. Therefore, user are not allowed to
# change the value of input series inside the window
# function. It is rare that user needs to modify the
# input series in the window function, and therefore,
# it is a reasonable restriction.
# Note: Calling reset_index on the slices will increase the
# cost of creating slices by about 100%. Therefore, for
# performance reasons we don't do it here.
slices = [s.iloc[begin_array[i] : end_array[i]] for s in series]
kw_slices = {
k: s.iloc[begin_array[i] : end_array[i]] for k, s in kw_series.items()
}
results.append(udf_func(*slices, **kw_slices))
result_series.append(pd.Series(results))
else:
raise PySparkRuntimeError(
errorClass="INVALID_WINDOW_BOUND_TYPE",
messageParameters={"window_bound_type": bound_type},
)
yield PandasToArrowConversion.from_pandas(
result_series,
self._return_schema,
timezone=self._runner_conf.timezone,
safecheck=self._runner_conf.safecheck,
arrow_cast=True,
prefers_large_types=self._runner_conf.use_large_var_types,
assign_cols_by_name=self._runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=self._runner_conf.int_to_decimal_coercion_enabled,
)
class ArrowBatchedUDFLegacyPandasHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
"""SQL_ARROW_BATCHED_UDF (Arrow-optimized ``@udf``) with the legacy pandas conversion:
convert each input RecordBatch to pandas Series, invoke each UDF row by row on their
values, and convert the results back to Arrow through pandas."""
eval_type = PythonEvalType.SQL_ARROW_BATCHED_UDF
variant_conf = "use_legacy_pandas_udf_conversion"
variant_value = True
_multi_udf = True
# run wraps the row-by-row evaluation in fail_on_stopiteration itself.
_fail_on_stopiteration = False
def __init__(
self, udf_descs: list[UDF], runner_conf: "RunnerConf", eval_conf: "EvalConf"
) -> None:
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
# --- UDF preparation ---
self._udf_infos = []
for udf, func in zip(udf_descs, self._funcs):
udf_return_type = udf.return_type
wrapped_func, args_kwargs_offsets = wrap_kwargs_support(
func, udf.args_offsets, udf.kwargs_offsets
)
zero_arg = len(args_kwargs_offsets) == 0
# Legacy coerces String/Binary for Arrow compatibility
coerce = (
str
if isinstance(udf_return_type, StringType)
else bytes
if isinstance(udf_return_type, BinaryType)
else None
)
self._udf_infos.append(
(
wrapped_func,
args_kwargs_offsets or (0,),
zero_arg,
udf_return_type,
coerce,
)
)
col_names = [f"_{i}" for i in range(len(udf_descs))]
self._return_schema = StructType(
[StructField(name, info[3]) for name, info in zip(col_names, self._udf_infos)]
)
def run(self, split_index: int, data: "Iterator[pa.RecordBatch]") -> "Iterator[pa.RecordBatch]":
import pandas as pd
runner_conf = self._runner_conf
@fail_on_stopiteration
def _evaluate_batch_udf_legacy(udf_func: Any, rows: list[tuple]) -> list[Any]:
if runner_conf.arrow_concurrency_level <= 0:
return [udf_func(*row) for row in rows]
from concurrent.futures import ThreadPoolExecutor
with ThreadPoolExecutor(max_workers=runner_conf.arrow_concurrency_level) as pool:
return list(pool.map(lambda row: udf_func(*row), rows))
for input_batch in data:
# --- Input: Arrow -> pandas columns ---
pandas_columns = ArrowToPandasConversion.to_pandas(
input_batch,
timezone=runner_conf.timezone,
schema=self._eval_conf.input_type, # type: ignore[arg-type]
struct_in_pandas="row",
ndarray_as_list=True,
prefer_int_ext_dtype=runner_conf.prefer_int_ext_dtype,
df_for_struct=False,
)
num_rows = len(pandas_columns[0]) if pandas_columns else input_batch.num_rows
if not pandas_columns:
pandas_columns = [pd.Series([_NoValue] * num_rows)]
# --- Process: evaluate each UDF row-by-row ---
result_series = []
for udf_func, offsets, zero_arg, _, coerce in self._udf_infos:
rows = (
[() for _ in range(num_rows)]
if zero_arg
else list(
zip(*[pandas_columns[o].tolist() for o in offsets]) # type: ignore[operator]
)
)
results = _evaluate_batch_udf_legacy(udf_func, rows)
verify_result_row_count(len(results), num_rows)
if coerce:
results = [coerce(v) if v is not None else v for v in results]
result_series.append(pd.Series(results))
# --- Output: pandas -> Arrow ---
yield PandasToArrowConversion.from_pandas(
result_series,
self._return_schema,
timezone=runner_conf.timezone,
safecheck=runner_conf.safecheck,
arrow_cast=True,
prefers_large_types=runner_conf.use_large_var_types,
assign_cols_by_name=runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=runner_conf.int_to_decimal_coercion_enabled,
)