blob: e179f4b68a1a1dec314318f0e4a78a87d2e6faab [file]
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Handlers for the user-defined table function (UDTF) eval types, and the wrappers that
implement TABLE argument PARTITION BY semantics on top of a UDTF instance.
Each handler receives the single ``UDTF`` record the worker reads, and instantiates the
UDTF from it.
pyarrow is imported lazily (inside the handlers and the type-checking block) so the module
stays importable and its handlers register without pyarrow installed.
"""
from __future__ import annotations
import dataclasses
import inspect
import itertools
from abc import ABCMeta, abstractmethod
from collections.abc import Iterable, Iterator
from typing import TYPE_CHECKING, Any, Callable, ClassVar, Optional, Tuple
from pyspark.errors import PySparkRuntimeError, PySparkTypeError, PySparkValueError
from pyspark.eval_handlers._base import BatchEvalTypeHandler, EvalTypeHandler
from pyspark.eval_handlers._typing import UDTF, InputBatch, OutputBatch, UDTFInstance
from pyspark.eval_handlers.utils import wrap_kwargs_support
from pyspark.eval_handlers.verification import verify_arrow_result, verify_pandas_result
from pyspark.serializers import (
BatchedSerializer,
CPickleSerializer,
Serializer,
)
from pyspark.sql.conversion import (
ArrowBatchTransformer,
ArrowTableToRowsConversion,
ArrowToPandasConversion,
LocalDataToArrowConversion,
PandasToArrowConversion,
)
from pyspark.sql.functions import SkipRestOfInputTableException
from pyspark.sql.pandas.types import to_arrow_type
from pyspark.sql.types import (
ArrayType,
DataType,
MapType,
Row,
StructField,
StructType,
_create_row,
)
from pyspark.util import PythonEvalType
if TYPE_CHECKING:
import pandas as pd
import pyarrow as pa
from pyspark.worker_util import EvalConf, RunnerConf
class PartitionedUDTF(metaclass=ABCMeta):
"""
Base for the wrappers that implement TABLE argument PARTITION BY semantics on top of a UDTF.
A wrapper owns one UDTF instance per partition: when ``eval`` sees a new partition it calls
``terminate`` on the current instance and replaces it with a fresh one. A
``SkipRestOfInputTableException`` from ``eval`` skips the rest of the current partition only.
"""
def __init__(self, create_udtf: Callable, partition_child_indexes: list):
self._create_udtf: Callable = create_udtf
self._udtf = create_udtf()
self._partition_child_indexes: list = partition_child_indexes
self._eval_raised_skip_rest_of_input_table: bool = False
@abstractmethod
def eval(self, *args: Any, **kwargs: Any) -> Iterator:
"""Evaluate the input, calling the per-partition UDTF once per partition it covers."""
def terminate(self) -> Iterator:
if hasattr(self._udtf, "terminate"):
return self._udtf.terminate()
return iter(())
def cleanup(self) -> None:
if hasattr(self._udtf, "cleanup"):
self._udtf.cleanup()
def _start_next_partition(self) -> Iterator:
"""Terminate the current partition's UDTF, yielding its results, and start a new one."""
if hasattr(self._udtf, "terminate"):
result = self._udtf.terminate()
if result is not None:
yield from result
self._udtf = self._create_udtf()
self._eval_raised_skip_rest_of_input_table = False
def _eval_partition(self, *args: Any, **kwargs: Any) -> Iterator:
"""Call the current partition's UDTF ``eval``, yielding its results."""
try:
result = self._udtf.eval(*args, **kwargs)
if result is not None:
yield from result
except SkipRestOfInputTableException:
# Skip the rest of the rows in the current partition: callers stop calling 'eval'
# until they see a change in the partition boundaries.
self._eval_raised_skip_rest_of_input_table = True
class UDTFWithPartitions(PartitionedUDTF):
"""
This implements the logic of a UDTF that accepts an input TABLE argument with one or more
PARTITION BY expressions.
For example, let's assume we have a table like:
CREATE TABLE t (c1 INT, c2 INT) USING delta;
Then for the following queries:
SELECT * FROM my_udtf(TABLE (t) PARTITION BY c1, c2);
The partition_child_indexes will be: 0, 1.
SELECT * FROM my_udtf(TABLE (t) PARTITION BY c1, c2 + 4);
The partition_child_indexes will be: 0, 2 (where we add a projection for "c2 + 4").
"""
def __init__(self, create_udtf: Callable, partition_child_indexes: list):
"""
Creates a new instance of this class to wrap the provided UDTF with another one that
checks the values of projected partitioning expressions on consecutive rows to figure
out when the partition boundaries change.
Parameters
----------
create_udtf: function
Function to create a new instance of the UDTF to be invoked.
partition_child_indexes: list
List of integers identifying zero-based indexes of the columns of the input table
that contain projected partitioning expressions. This class will inspect these
values for each pair of consecutive input rows. When they change, this indicates
the boundary between two partitions, and we will invoke the 'terminate' method on
the UDTF class instance and then destroy it and create a new one to implement the
desired partitioning semantics.
"""
super().__init__(create_udtf, partition_child_indexes)
self._prev_arguments: list = list()
def eval(self, *args: Any, **kwargs: Any) -> Iterator:
changed_partitions = self._check_partition_boundaries(list(args) + list(kwargs.values()))
if changed_partitions:
yield from self._start_next_partition()
if self._udtf.eval is not None and not self._eval_raised_skip_rest_of_input_table:
# Filter the arguments to exclude projected PARTITION BY values added by Catalyst.
filtered_args = [self._remove_partition_by_exprs(arg) for arg in args]
filtered_kwargs = {
key: self._remove_partition_by_exprs(value) for (key, value) in kwargs.items()
}
yield from self._eval_partition(*filtered_args, **filtered_kwargs)
def _check_partition_boundaries(self, arguments: list) -> bool:
result = False
if len(self._prev_arguments) > 0:
cur_table_arg = self._get_table_arg(arguments)
prev_table_arg = self._get_table_arg(self._prev_arguments)
cur_partitions_args = []
prev_partitions_args = []
for i in self._partition_child_indexes:
cur_partitions_args.append(cur_table_arg[i])
prev_partitions_args.append(prev_table_arg[i])
result = any(k != v for k, v in zip(cur_partitions_args, prev_partitions_args))
self._prev_arguments = arguments
return result
def _get_table_arg(self, inputs: list) -> Row:
return [x for x in inputs if type(x) is Row][0]
def _remove_partition_by_exprs(self, arg: Any) -> Any:
if isinstance(arg, Row):
new_row_keys = []
new_row_values = []
for i, (key, value) in enumerate(zip(arg.__fields__, arg)):
if i not in self._partition_child_indexes:
new_row_keys.append(key)
new_row_values.append(value)
return _create_row(new_row_keys, new_row_values)
else:
return arg
class ArrowUDTFWithPartition(PartitionedUDTF):
"""
Implements logic for an Arrow UDTF (SQL_ARROW_UDTF) that accepts a TABLE argument
with one or more PARTITION BY expressions.
Arrow UDTFs receive data as PyArrow RecordBatch objects instead of individual Row
objects. This wrapper ensures the UDTF's eval() method is called separately for each
unique partition key value combination.
How Catalyst handles PARTITION BY and ORDER BY:
------------------------------------------------
When a UDTF is called with PARTITION BY and/or ORDER BY clauses, Catalyst adds
operations to the physical plan to ensure correct data organization:
Example SQL:
SELECT * FROM my_udtf(TABLE(t) PARTITION BY key1, key2 ORDER BY value DESC)
Physical Plan generated by Catalyst:
1. Project: Adds partition_by_0 = key1, partition_by_1 = key2 columns
2. Exchange: hashpartitioning(partition_by_0, partition_by_1, 200)
- Shuffles data so rows with same partition keys go to same worker
3. Sort: [partition_by_0 ASC, partition_by_1 ASC, value DESC], local=true
- First sorts by partition keys to group them together
- Then sorts by ORDER BY expressions within each partition
- Local sort (not global) within each worker's data
4. Project: Creates struct with all columns including partition_by_* columns
5. ArrowEvalPythonUDTF: Executes this Python UDTF wrapper
Key guarantee: After the Sort operation, all rows with the same partition key
values are contiguous within each RecordBatch, allowing efficient boundary detection.
Example queries:
SELECT * FROM my_udtf(TABLE (t) PARTITION BY c1);
partition_child_indexes: [2] (refers to partition_by_0 column at index 2)
SELECT * FROM my_udtf(TABLE (t) PARTITION BY c1, c2);
partition_child_indexes: [2, 3] (partition_by_0 and partition_by_1 columns)
SELECT * FROM my_udtf(TABLE (t) PARTITION BY c1, c2 + 4);
partition_child_indexes: 0, 2 (adds a projection for "c2 + 4").
"""
def __init__(self, create_udtf: Callable, partition_child_indexes: list):
"""
Create a new instance that wraps the provided Arrow UDTF with partitioning
logic.
Parameters
----------
create_udtf: function
Function that creates a new instance of the Arrow UDTF to invoke.
partition_child_indexes: list
Zero-based indexes of input-table columns that contain projected
partitioning expressions.
"""
super().__init__(create_udtf, partition_child_indexes)
# Track last partition key from previous batch
self._last_partition_key: Optional[Tuple[Any, ...]] = None
def eval(self, *args: Any, **kwargs: Any) -> Iterator:
"""Handle partitioning logic for Arrow UDTFs that receive RecordBatch objects."""
import pyarrow as pa
# Get the original batch with partition columns
original_batch = self._get_table_arg(list(args) + list(kwargs.values()))
if not isinstance(original_batch, pa.RecordBatch):
# Arrow UDTFs with PARTITION BY must have a TABLE argument that
# results in a PyArrow RecordBatch
raise PySparkRuntimeError(
errorClass="INVALID_ARROW_UDTF_TABLE_ARGUMENT",
messageParameters={
"actual_type": (
str(type(original_batch)) if original_batch is not None else "None"
)
},
)
# Remove partition columns to get the filtered arguments
filtered_args = [self._remove_partition_by_exprs(arg) for arg in args]
filtered_kwargs = {
key: self._remove_partition_by_exprs(value) for (key, value) in kwargs.items()
}
# Get the filtered RecordBatch (without partition columns)
filtered_batch = self._get_table_arg(filtered_args + list(filtered_kwargs.values()))
# Process the RecordBatch by partitions
yield from self._process_arrow_batch_by_partitions(
original_batch, filtered_batch, filtered_args, filtered_kwargs
)
def _process_arrow_batch_by_partitions(
self,
original_batch: pa.RecordBatch,
filtered_batch: pa.RecordBatch,
filtered_args: list,
filtered_kwargs: dict,
) -> Iterator:
"""Process an Arrow RecordBatch that may contain multiple partition key values.
When using PARTITION BY with Arrow UDTFs, a single RecordBatch from Spark may contain
rows with different partition key values. For example, with 10 distinct partition keys
and 2 workers, each worker might receive a batch containing 5 different partition key
values.
According to UDTF PARTITION BY semantics, the UDTF's eval() method must be called
separately for each unique partition key value, not for the entire batch. This method
handles splitting the batch by partition boundaries and calling the UDTF appropriately.
The implementation leverages two key properties:
1. Catalyst guarantees rows with the same partition key are contiguous (pre-sorted)
2. Arrow's columnar format allows efficient boundary detection
Parameters:
-----------
original_batch : pa.RecordBatch
The original batch including partition columns, used for detecting boundaries
filtered_batch : pa.RecordBatch
The batch with partition columns removed, to be passed to the UDTF
filtered_args : list
Arguments with partition columns filtered out
filtered_kwargs : dict
Keyword arguments with partition columns filtered out
Yields:
-------
Iterator of pa.Table objects returned by the UDTF's eval() method
"""
import pyarrow as pa
# This class should only be used when partition_child_indexes is non-empty
assert self._partition_child_indexes, (
"ArrowUDTFWithPartition should only be instantiated when "
"len(partition_child_indexes) > 0"
)
# Detect partition boundaries.
boundaries = self._detect_partition_boundaries(original_batch)
# Process each contiguous partition
for i in range(len(boundaries) - 1):
start_idx = boundaries[i]
end_idx = boundaries[i + 1]
# Get the partition key for this segment
partition_key = tuple(
original_batch.column(idx)[start_idx].as_py()
for idx in self._partition_child_indexes
)
# Check if this is a continuation of the previous batch's partition
# TODO: This check is only necessary for the first boundary in each batch.
# The following boundaries are always for new partitions within the same batch.
# This could be optimized by only checking i == 0.
is_new_partition = (
self._last_partition_key is not None and partition_key != self._last_partition_key
)
if is_new_partition:
# Previous partition ended: terminate it and start a new UDTF instance
yield from self._start_next_partition()
# Slice the filtered batch for this partition
partition_batch = filtered_batch.slice(start_idx, end_idx - start_idx)
# Update the last partition key
self._last_partition_key = partition_key
# Update filtered args to use the partition batch
partition_filtered_args = []
for arg in filtered_args:
if isinstance(arg, pa.RecordBatch):
partition_filtered_args.append(partition_batch)
else:
partition_filtered_args.append(arg)
partition_filtered_kwargs = {}
for key, value in filtered_kwargs.items():
if isinstance(value, pa.RecordBatch):
partition_filtered_kwargs[key] = partition_batch
else:
partition_filtered_kwargs[key] = value
# Call the UDTF with this partition's data
if not self._eval_raised_skip_rest_of_input_table:
yield from self._eval_partition(
*partition_filtered_args, **partition_filtered_kwargs
)
# Don't terminate here - let the next batch or final terminate handle it
def _get_table_arg(self, inputs: list) -> Optional[pa.RecordBatch]:
"""Get the table argument (RecordBatch) from the inputs list.
For Arrow UDTFs with TABLE arguments, we can guarantee the table argument
will be a pa.RecordBatch, not a Row.
"""
import pyarrow as pa
# Find all RecordBatch arguments
batches = [arg for arg in inputs if isinstance(arg, pa.RecordBatch)]
if len(batches) == 0:
# No RecordBatch found - this shouldn't happen for Arrow UDTFs with TABLE arguments
return None
elif len(batches) == 1:
return batches[0]
else:
# Multiple RecordBatch arguments found - this is unexpected
raise RuntimeError(
f"Expected exactly one pa.RecordBatch argument for TABLE parameter, "
f"but found {len(batches)}. Received types: "
f"{[type(arg).__name__ for arg in inputs]}"
)
def _detect_partition_boundaries(self, batch: pa.RecordBatch) -> list:
"""
Efficiently detect partition boundaries in a batch with contiguous partitions.
Since Catalyst ensures rows with the same partition key are contiguous,
we only need to find where partition values change.
Returns:
List of indices where each partition starts, plus the total row count.
For example: [0, 3, 8, 10] means partitions are rows [0:3), [3:8), [8:10)
"""
boundaries = [0] # First partition starts at index 0
if batch.num_rows <= 1:
boundaries.append(batch.num_rows)
return boundaries
# Get partition column arrays
partition_arrays = [batch.column(i) for i in self._partition_child_indexes]
# Find boundaries by comparing consecutive rows
for row_idx in range(1, batch.num_rows):
# Check if any partition column changed from previous row
partition_changed = False
for col_array in partition_arrays:
if col_array[row_idx].as_py() != col_array[row_idx - 1].as_py():
partition_changed = True
break
if partition_changed:
boundaries.append(row_idx)
boundaries.append(batch.num_rows) # Last boundary at end
return boundaries
def _remove_partition_by_exprs(self, arg: Any) -> Any:
"""
Remove partition columns from the RecordBatch argument.
Why this is needed:
When a UDTF is called with TABLE(t) PARTITION BY expressions, Catalyst transforms
the data:
1. Adds complex partition expressions as new columns
(e.g., "c2 + 4" becomes a new column)
2. Repartitions data by partition columns using hash partitioning
3. Sends ALL columns (including partition columns) to the Python worker
Partition columns serve two purposes:
- Routing: decide which worker processes which partition
- Boundary detection: know when one partition ends and another begins
However, the user's UDTF should only receive the actual table data, not the
partition columns. This method filters out partition columns before passing
data to the user's UDTF eval() method.
Example:
- User writes: SELECT * FROM udtf(TABLE(t) PARTITION BY c1, c2)
- Catalyst sends: RecordBatch with [c1, c2, c3, c4],
partition_child_indexes=[0, 1]
- This method removes columns at indexes 0, 1 if they are pure partition columns
- UDTF.eval() receives: RecordBatch with only the non-partition columns
"""
import pyarrow as pa
if isinstance(arg, pa.RecordBatch):
# Remove partition columns from the RecordBatch
keep_indices = [
i for i in range(len(arg.schema.names)) if i not in self._partition_child_indexes
]
if keep_indices:
# Select only the columns we want to keep
keep_arrays = [arg.column(i) for i in keep_indices]
keep_names = [arg.schema.names[i] for i in keep_indices]
return pa.RecordBatch.from_arrays(keep_arrays, names=keep_names)
else:
# If no columns remain, return an empty RecordBatch with the same number of rows
return pa.RecordBatch.from_arrays([], schema=pa.schema([]), num_rows=arg.num_rows)
# For non-RecordBatch arguments (like scalar pa.Arrays), return unchanged
return arg
def build_null_checker(return_type: StructType) -> Optional[Callable[[Any], None]]:
def raise_(result_column_index: int) -> None:
raise PySparkRuntimeError(
errorClass="UDTF_EXEC_ERROR",
messageParameters={
"method_name": "eval' or 'terminate",
"error": f"Column {result_column_index} within a returned row had a "
+ "value of None, either directly or within array/struct/map "
+ "subfields, but the corresponding column type was declared as "
+ "non-nullable; please update the UDTF to return a non-None value at "
+ "this location or otherwise declare the column type as nullable.",
},
)
def checker(data_type: DataType, result_column_index: int) -> Optional[Callable[[Any], None]]:
if isinstance(data_type, ArrayType):
element_checker = checker(data_type.elementType, result_column_index)
contains_null = data_type.containsNull
if element_checker is None and contains_null:
return None
def check_array(arr: Any) -> None:
if isinstance(arr, list):
for e in arr:
if e is None:
if not contains_null:
raise_(result_column_index)
elif element_checker is not None:
element_checker(e)
return check_array
elif isinstance(data_type, MapType):
key_checker = checker(data_type.keyType, result_column_index)
value_checker = checker(data_type.valueType, result_column_index)
value_contains_null = data_type.valueContainsNull
if value_checker is None and value_contains_null:
def check_map(map: Any) -> None:
if isinstance(map, dict):
for k, v in map.items():
if k is None:
raise_(result_column_index)
elif key_checker is not None:
key_checker(k)
else:
def check_map(map: Any) -> None:
if isinstance(map, dict):
for k, v in map.items():
if k is None:
raise_(result_column_index)
elif key_checker is not None:
key_checker(k)
if v is None:
if not value_contains_null:
raise_(result_column_index)
elif value_checker is not None:
value_checker(v)
return check_map
elif isinstance(data_type, StructType):
field_checkers = [checker(f.dataType, result_column_index) for f in data_type]
nullables = [f.nullable for f in data_type]
if all(c is None for c in field_checkers) and all(nullables):
return None
def check_struct(struct: Any) -> None:
if isinstance(struct, tuple):
for value, checker, nullable in zip(struct, field_checkers, nullables):
if value is None:
if not nullable:
raise_(result_column_index)
elif checker is not None:
checker(value)
return check_struct
else:
return None
field_checkers = [checker(f.dataType, result_column_index=i) for i, f in enumerate(return_type)]
nullables = [f.nullable for f in return_type]
if all(c is None for c in field_checkers) and all(nullables):
return None
def check(row: Any) -> None:
if isinstance(row, tuple):
for i, (value, checker, nullable) in enumerate(zip(row, field_checkers, nullables)):
if value is None:
if not nullable:
raise_(i)
elif checker is not None:
checker(value)
return check
class UDTFHandlerBase(EvalTypeHandler[InputBatch, OutputBatch]):
"""Base for every UDTF eval type handler; ``__init__`` keeps the UDTF record as
``_udtf_desc`` and creates the UDTF instance as ``_udtf``."""
# Wraps the UDTF when its TABLE argument has PARTITION BY expressions.
partition_wrapper: ClassVar[type[PartitionedUDTF]] = UDTFWithPartitions
def _expected_udf(self) -> str:
return "UDTF"
def __init__(self, udf_descs: list[UDTF], runner_conf: RunnerConf, eval_conf: EvalConf) -> None:
"""Create, as ``_udtf``, the instance of the UDTF's class (wrapped in
``partition_wrapper`` when its TABLE argument has PARTITION BY expressions, its
constructor given the AnalyzeResult when it takes one), and check its 'eval' signature
against the call arguments."""
super().__init__(udf_descs, runner_conf, eval_conf)
self._udtf_desc: UDTF = udf_descs[0]
udtf = self._udtf_desc
if not isinstance(udtf.udtf_cls, type):
raise PySparkRuntimeError(
f"Invalid UDTF handler type. Expected a class (type 'type'), but "
f"got an instance of {type(udtf.udtf_cls).__name__}."
)
# Pick how to construct the UDTF from the count of named positional __init__ parameters
# after self. With an AnalyzeResult, one parameter receives a copy of it, zero means the
# class is called with no arguments, and more are an error; without one, any is an error.
handler: Callable[[], Any] = udtf.udtf_cls
udtf_init_args = inspect.getfullargspec(udtf.udtf_cls)
if udtf.analyze_result is not None:
if len(udtf_init_args.args) > 2:
raise PySparkRuntimeError(
errorClass="UDTF_CONSTRUCTOR_INVALID_IMPLEMENTS_ANALYZE_METHOD",
messageParameters={"name": udtf.name},
)
elif len(udtf_init_args.args) == 2:
analyze_result = udtf.analyze_result
def construct_udtf() -> Any:
# Here we pass the AnalyzeResult to the UDTF's __init__ method.
return udtf.udtf_cls(dataclasses.replace(analyze_result))
handler = construct_udtf
elif len(udtf_init_args.args) > 1:
raise PySparkRuntimeError(
errorClass="UDTF_CONSTRUCTOR_INVALID_NO_ANALYZE_METHOD",
messageParameters={"name": udtf.name},
)
# Instantiate the UDTF class.
try:
if len(udtf.partition_child_indexes) > 0:
instance = self.partition_wrapper(handler, udtf.partition_child_indexes)
else:
instance = handler()
except Exception as e:
raise PySparkRuntimeError(
errorClass="UDTF_EXEC_ERROR",
messageParameters={"method_name": "__init__", "error": str(e)},
)
# Validate the UDTF
if not hasattr(instance, "eval"):
raise PySparkRuntimeError(
"Failed to execute the user defined table function because it has not "
"implemented the 'eval' method. Please add the 'eval' method and try "
"the query again."
)
# Check that the arguments provided to the UDTF call match the expected parameters
# defined in the 'eval' method signature.
try:
inspect.signature(instance.eval).bind(*udtf.args_offsets, **udtf.kwargs_offsets)
except TypeError as e:
raise PySparkRuntimeError(
errorClass="UDTF_EVAL_METHOD_ARGUMENTS_DO_NOT_MATCH_SIGNATURE",
messageParameters={"name": udtf.name, "reason": str(e)},
) from None
self._udtf: UDTFInstance = instance
class UDTFEvalTypeHandler(UDTFHandlerBase[InputBatch, OutputBatch]):
"""Base for the Arrow UDTF eval types.
``run`` calls ``_eval_batch`` per input batch, then ``_eval_terminate`` once at the end (also
when ``eval`` raised ``SkipRestOfInputTableException``), and always calls the UDTF's
``cleanup``. Subclasses implement the two hooks and pick the serializer.
"""
def __init__(self, udf_descs: list[UDTF], runner_conf: RunnerConf, eval_conf: EvalConf) -> None:
super().__init__(udf_descs, runner_conf, eval_conf)
self._eval_method, self._args_kwargs_offsets = wrap_kwargs_support(
self._udtf.eval, self._udtf_desc.args_offsets, self._udtf_desc.kwargs_offsets
)
self._terminate_method = getattr(self._udtf, "terminate", None)
self._cleanup_method = getattr(self._udtf, "cleanup", None)
@abstractmethod
def _eval_batch(self, batch: InputBatch) -> Iterator[OutputBatch]:
"""Call the UDTF's ``eval`` on one input batch and yield the converted results."""
@abstractmethod
def _eval_terminate(self) -> Iterator[OutputBatch]:
"""Call the UDTF's ``terminate`` and yield the converted results."""
def run(self, split_index: int, data: Iterator[InputBatch]) -> Iterator[OutputBatch]:
terminate = self._terminate_method
cleanup = self._cleanup_method
try:
for batch in data:
yield from self._eval_batch(batch)
if terminate is not None:
yield from self._eval_terminate()
except SkipRestOfInputTableException:
if terminate is not None:
yield from self._eval_terminate()
finally:
if cleanup is not None:
cleanup()
class TableUDTFHandler(UDTFHandlerBase[Any, tuple]):
"""SQL_TABLE_UDF, the Python UDTF: input rows arrive pickled one at a time and the UDTF's
``eval`` is called once per row; the results of each call are converted to the internal
representation and returned to the executor as one tuple."""
eval_type = PythonEvalType.SQL_TABLE_UDF
def __init__(self, udf_descs: list[UDTF], runner_conf: RunnerConf, eval_conf: EvalConf) -> None:
super().__init__(udf_descs, runner_conf, eval_conf)
self._check_output_row_against_schema = build_null_checker(self._udtf_desc.return_type)
eval_func_kwargs_support, self._args_kwargs_offsets = wrap_kwargs_support(
self._udtf.eval, self._udtf_desc.args_offsets, self._udtf_desc.kwargs_offsets
)
self._eval = self._wrap_udtf(eval_func_kwargs_support, self._udtf_desc.return_type)
self._terminate: Optional[Callable[..., tuple]]
if hasattr(self._udtf, "terminate"):
self._terminate = self._wrap_udtf(
getattr(self._udtf, "terminate"), self._udtf_desc.return_type
)
else:
self._terminate = None
self._cleanup = getattr(self._udtf, "cleanup") if hasattr(self._udtf, "cleanup") else None
@property
def serializer(self) -> Serializer:
# Each row is a group so do not batch but send one by one.
return BatchedSerializer(CPickleSerializer(), 1)
def _wrap_udtf(self, f: Callable, return_type: StructType) -> Callable[..., tuple]:
assert return_type.needConversion()
toInternal = return_type.toInternal
return_type_size = len(return_type)
check_output_row_against_schema = self._check_output_row_against_schema
def verify_and_convert_result(result: Any) -> Any:
if result is not None:
if hasattr(result, "__UDT__"):
# UDT object should not be returned directly.
raise PySparkRuntimeError(
errorClass="UDTF_INVALID_OUTPUT_ROW_TYPE",
messageParameters={
"type": type(result).__name__,
"func": f.__name__,
},
)
if hasattr(result, "__len__") and len(result) != return_type_size:
raise PySparkRuntimeError(
errorClass="UDTF_RETURN_SCHEMA_MISMATCH",
messageParameters={
"expected": str(return_type_size),
"actual": str(len(result)),
"func": f.__name__,
},
)
if not (isinstance(result, (list, dict, tuple)) or hasattr(result, "__dict__")):
raise PySparkRuntimeError(
errorClass="UDTF_INVALID_OUTPUT_ROW_TYPE",
messageParameters={
"type": type(result).__name__,
"func": f.__name__,
},
)
if check_output_row_against_schema is not None:
check_output_row_against_schema(result)
return toInternal(result)
# Evaluate the function and return a tuple back to the executor.
def evaluate(*a: Any) -> tuple:
try:
res = f(*a)
except SkipRestOfInputTableException:
raise
except Exception as e:
raise PySparkRuntimeError(
errorClass="UDTF_EXEC_ERROR",
messageParameters={"method_name": f.__name__, "error": str(e)},
)
if res is None:
# If the function returns None or does not have an explicit return statement,
# an empty tuple is returned to the executor.
# This is because directly constructing tuple(None) results in an exception.
return tuple()
if not isinstance(res, Iterable):
raise PySparkRuntimeError(
errorClass="UDTF_RETURN_NOT_ITERABLE",
messageParameters={
"type": type(res).__name__,
"func": f.__name__,
},
)
# If the function returns a result, we map it to the internal representation and
# returns the results as a tuple.
return tuple(map(verify_and_convert_result, res))
return evaluate
def run(self, split_index: int, data: Iterator[Any]) -> Iterator[tuple]:
eval = self._eval
terminate = self._terminate
cleanup = self._cleanup
args_kwargs_offsets = self._args_kwargs_offsets
try:
for a in data:
yield eval(*[a[o] for o in args_kwargs_offsets])
if terminate is not None:
yield terminate()
except SkipRestOfInputTableException:
if terminate is not None:
yield terminate()
finally:
if cleanup is not None:
cleanup()
class ArrowUDTFHandler(
BatchEvalTypeHandler["pa.RecordBatch"],
UDTFEvalTypeHandler["pa.RecordBatch", "pa.RecordBatch"],
):
"""SQL_ARROW_UDTF: the UDTF's ``eval`` receives ``pa.Array`` arguments, with TABLE
arguments flattened into ``pa.RecordBatch``, and returns an iterable of
``pa.Table``/``pa.RecordBatch``, coerced to the declared schema."""
eval_type = PythonEvalType.SQL_ARROW_UDTF
partition_wrapper = ArrowUDTFWithPartition
def __init__(self, udf_descs: list[UDTF], runner_conf: RunnerConf, eval_conf: EvalConf) -> None:
import pyarrow as pa
super().__init__(udf_descs, runner_conf, eval_conf)
arrow_return_type = to_arrow_type(
self._udtf_desc.return_type,
timezone="UTC",
prefers_large_types=runner_conf.use_large_var_types,
)
self._return_type_size = len(self._udtf_desc.return_type)
self._target_schema = pa.schema(list(arrow_return_type))
self._table_arg_offsets = (
set(eval_conf.table_arg_offsets) if eval_conf.table_arg_offsets else set()
)
def _verify_result(self, result: pa.RecordBatch, method_name: str) -> pa.RecordBatch:
# Validate the output schema when the result has columns
if result.num_columns != self._return_type_size:
raise PySparkRuntimeError(
errorClass="UDTF_RETURN_SCHEMA_MISMATCH",
messageParameters={
"expected": str(self._return_type_size),
"actual": str(result.num_columns),
"func": method_name,
},
)
return result
def _convert_to_arrow(self, res: Any, method_name: str) -> Iterator[pa.RecordBatch]:
import pyarrow as pa
# Check whether the result of a PyArrow UDTF is iterable before processing
if res is None:
res = iter([])
elif not isinstance(res, Iterable):
raise PySparkRuntimeError(
errorClass="UDTF_RETURN_NOT_ITERABLE",
messageParameters={
"type": type(res).__name__,
"func": method_name,
},
)
# Handle PyArrow Tables/RecordBatches directly
is_empty = True
for item in res:
is_empty = False
if isinstance(item, pa.Table):
yield from item.to_batches()
elif isinstance(item, pa.RecordBatch):
yield item
else:
# Arrow UDTF should only return Arrow types (RecordBatch/Table)
raise PySparkRuntimeError(
errorClass="UDTF_ARROW_TYPE_CONVERSION_ERROR",
messageParameters={},
)
if is_empty:
yield pa.RecordBatch.from_pylist([], schema=self._target_schema)
def _evaluate(self, method: Callable, *args: pa.RecordBatch) -> Iterator[pa.RecordBatch]:
# Wrap the exception thrown from the UDTF in a PySparkRuntimeError.
try:
res = method(*args)
except SkipRestOfInputTableException:
raise
except Exception as e:
raise PySparkRuntimeError(
errorClass="UDTF_EXEC_ERROR",
messageParameters={"method_name": method.__name__, "error": str(e)},
)
for batch in self._convert_to_arrow(res, method.__name__):
coerced = ArrowBatchTransformer.enforce_schema(
self._verify_result(batch, method.__name__), self._target_schema, safecheck=True
)
yield ArrowBatchTransformer.wrap_struct(coerced)
def _eval_batch(self, batch: pa.RecordBatch) -> Iterator[pa.RecordBatch]:
# Pre-processing: for each column, flatten struct columns at
# table_arg_offsets into RecordBatch, keep other columns as Array.
table_arg_offsets = self._table_arg_offsets
columns = [
(
ArrowBatchTransformer.flatten_struct(batch, column_index=i)
if i in table_arg_offsets
else batch.column(i)
)
for i in range(batch.num_columns)
]
# For PyArrow UDTFs, pass RecordBatches directly (no row conversion needed)
yield from self._evaluate(
self._eval_method, *[columns[o] for o in self._args_kwargs_offsets]
)
def _eval_terminate(self) -> Iterator[pa.RecordBatch]:
assert self._terminate_method is not None
yield from self._evaluate(self._terminate_method)
class ArrowTableUDTFHandler(
BatchEvalTypeHandler["pa.RecordBatch"],
UDTFEvalTypeHandler["pa.RecordBatch", "pa.RecordBatch"],
):
"""Base for SQL_ARROW_TABLE_UDF, the Arrow-optimized Python UDTF: each input RecordBatch is
converted to per-column values and the UDTF's ``eval`` is called once per input row. The
variants differ in how the values are converted, selected by
``spark.sql.legacy.execution.pythonUDTF.pandas.conversion.enabled``. Subclasses implement
``_to_columns`` and ``_convert_result``."""
def __init__(self, udf_descs: list[UDTF], runner_conf: RunnerConf, eval_conf: EvalConf) -> None:
super().__init__(udf_descs, runner_conf, eval_conf)
self._return_type_size = len(self._udtf_desc.return_type)
self._check_output_row_against_schema = build_null_checker(self._udtf_desc.return_type)
@abstractmethod
def _to_columns(self, batch: pa.RecordBatch) -> list:
"""Convert the input batch into one sequence of per-row values per column."""
@abstractmethod
def _convert_result(self, res: Any, method_name: str) -> Iterator[pa.RecordBatch]:
"""Convert the result of one ``eval``/``terminate`` call into output batches."""
def _evaluate_rows(
self, method: Callable, *args: list, num_rows: int = 1
) -> Iterator[pa.RecordBatch]:
rows = itertools.repeat((), num_rows) if len(args) == 0 else zip(*args)
for row in rows:
# Wrap the exception thrown from the UDTF in a PySparkRuntimeError.
try:
res = method(*row)
except SkipRestOfInputTableException:
raise
except Exception as e:
raise PySparkRuntimeError(
errorClass="UDTF_EXEC_ERROR",
messageParameters={"method_name": method.__name__, "error": str(e)},
)
yield from self._convert_result(res, method.__name__)
def _eval_batch(self, batch: pa.RecordBatch) -> Iterator[pa.RecordBatch]:
columns = self._to_columns(batch)
yield from self._evaluate_rows(
self._eval_method,
*[columns[o] for o in self._args_kwargs_offsets],
num_rows=batch.num_rows,
)
def _eval_terminate(self) -> Iterator[pa.RecordBatch]:
assert self._terminate_method is not None
yield from self._evaluate_rows(self._terminate_method)
class ArrowTableUDTFLegacyPandasHandler(ArrowTableUDTFHandler):
"""SQL_ARROW_TABLE_UDF with the legacy pandas conversion: the input is converted to pandas
Series, and each ``eval`` result is collected into a ``pd.DataFrame`` and converted back to
Arrow as a single ``_0`` struct column."""
eval_type = PythonEvalType.SQL_ARROW_TABLE_UDF
variant_conf = "use_legacy_pandas_udtf_conversion"
variant_value = True
def __init__(self, udf_descs: list[UDTF], runner_conf: RunnerConf, eval_conf: EvalConf) -> None:
import pandas # noqa: F401
super().__init__(udf_descs, runner_conf, eval_conf)
# The output pandas DataFrame is converted as a single struct column named
# "_0" against this schema.
self._output_schema = StructType([StructField("_0", self._udtf_desc.return_type)])
def _verify_result(self, result: Any, method_name: str) -> Any:
import pandas as pd
if not isinstance(result, pd.DataFrame):
raise PySparkTypeError(
errorClass="INVALID_ARROW_UDTF_RETURN_TYPE",
messageParameters={
"return_type": type(result).__name__,
"value": str(result),
"func": method_name,
},
)
# Validate the output schema when the result dataframe has either output
# rows or columns. Note that we avoid using `df.empty` here because the
# result dataframe may contain an empty row. For example, when a UDTF is
# defined as follows: def eval(self): yield tuple().
if len(result) > 0 or len(result.columns) > 0:
if len(result.columns) != self._return_type_size:
raise PySparkRuntimeError(
errorClass="UDTF_RETURN_SCHEMA_MISMATCH",
messageParameters={
"expected": str(self._return_type_size),
"actual": str(len(result.columns)),
"func": method_name,
},
)
# Verify the type and the schema of the result.
verify_pandas_result(
result,
self._udtf_desc.return_type,
assign_cols_by_name=False,
truncate_return_schema=False,
)
return result
def _check_return_value(self, res: Any, method_name: str) -> Iterator:
# Check whether the result of an arrow UDTF is iterable before
# using it to construct a pandas DataFrame.
if res is not None:
if not isinstance(res, Iterable):
raise PySparkRuntimeError(
errorClass="UDTF_RETURN_NOT_ITERABLE",
messageParameters={
"type": type(res).__name__,
"func": method_name,
},
)
check_output_row_against_schema = self._check_output_row_against_schema
if check_output_row_against_schema is not None:
for row in res:
if row is not None:
check_output_row_against_schema(row)
yield row
else:
yield from res
def _convert_df_to_arrow(self, result: pd.DataFrame) -> pa.RecordBatch:
# Convert the output pandas DataFrame into a single "_0" struct column,
# applying the legacy pandas-to-Arrow coercions.
runner_conf = self._runner_conf
return PandasToArrowConversion.from_pandas(
[result],
self._output_schema,
timezone=runner_conf.timezone,
safecheck=runner_conf.safecheck,
arrow_cast=True,
assign_cols_by_name=False,
int_to_decimal_coercion_enabled=runner_conf.int_to_decimal_coercion_enabled,
ignore_unexpected_complex_type_values=True,
use_legacy_error_handling=True,
)
def _to_columns(self, batch: pa.RecordBatch) -> list:
# Deserialize the Arrow batch into a list of pandas Series (one per
# input column).
return ArrowToPandasConversion.to_pandas(
batch,
timezone=self._runner_conf.timezone,
schema=self._eval_conf.input_type, # type: ignore[arg-type]
struct_in_pandas="row",
ndarray_as_list=True,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
df_for_struct=False,
)
def _convert_result(self, res: Any, method_name: str) -> Iterator[pa.RecordBatch]:
import pandas as pd
result = self._verify_result(
pd.DataFrame(list(self._check_return_value(res, method_name))), method_name
)
yield self._convert_df_to_arrow(result)
class ArrowTableUDTFArrowHandler(ArrowTableUDTFHandler):
"""SQL_ARROW_TABLE_UDF with the Arrow conversion: each input column is converted to Python
values, and each ``eval`` result's rows are converted to Arrow against the declared
schema and wrapped as a single struct column."""
eval_type = PythonEvalType.SQL_ARROW_TABLE_UDF
variant_conf = "use_legacy_pandas_udtf_conversion"
variant_value = False
def __init__(self, udf_descs: list[UDTF], runner_conf: RunnerConf, eval_conf: EvalConf) -> None:
import pyarrow # noqa: F401
super().__init__(udf_descs, runner_conf, eval_conf)
self._arrow_return_type = to_arrow_type(
self._udtf_desc.return_type,
timezone="UTC",
prefers_large_types=runner_conf.use_large_var_types,
)
self._converters = [
ArrowTableToRowsConversion._create_converter(
f.dataType,
none_on_identity=True,
binary_as_bytes=runner_conf.binary_as_bytes,
)
for f in eval_conf.input_type # type: ignore[union-attr]
]
def _verify_result(self, result: pa.Table, method_name: str) -> pa.Table:
import pyarrow as pa
if not isinstance(result, pa.Table):
raise PySparkTypeError(
errorClass="INVALID_ARROW_UDTF_RETURN_TYPE",
messageParameters={
"return_type": type(result).__name__,
"value": str(result),
"func": method_name,
},
)
# Validate the output schema when the result dataframe has either output
# rows or columns. Note that we avoid using `df.empty` here because the
# result dataframe may contain an empty row. For example, when a UDTF is
# defined as follows: def eval(self): yield tuple().
if result.num_rows > 0 or result.num_columns > 0:
if result.num_columns != self._return_type_size:
raise PySparkRuntimeError(
errorClass="UDTF_RETURN_SCHEMA_MISMATCH",
messageParameters={
"expected": str(self._return_type_size),
"actual": str(result.num_columns),
"func": method_name,
},
)
# Verify the type and the schema of the result.
verify_arrow_result(
result,
assign_cols_by_name=False,
expected_cols_and_types=[(field.name, field.type) for field in self._arrow_return_type],
)
return result
def _check_return_value(self, res: Any, method_name: str) -> Iterator:
# Check whether the result of an arrow UDTF is iterable before
# using it to construct a pandas DataFrame.
if res is not None:
if not isinstance(res, Iterable):
raise PySparkRuntimeError(
errorClass="UDTF_RETURN_NOT_ITERABLE",
messageParameters={
"type": type(res).__name__,
"func": method_name,
},
)
return_type_size = self._return_type_size
check_output_row_against_schema = self._check_output_row_against_schema
for row in res:
if not isinstance(row, tuple) and return_type_size == 1:
row = (row,)
if check_output_row_against_schema is not None:
if row is not None:
check_output_row_against_schema(row)
yield row
def _convert_rows_to_arrow(self, data: Iterable, method_name: str) -> list[pa.RecordBatch]:
import pyarrow as pa
arrow_return_type = self._arrow_return_type
data = list(self._check_return_value(data, method_name))
if len(data) == 0:
# Return one empty RecordBatch to match the left side of the lateral join
return [pa.RecordBatch.from_pylist(data, schema=pa.schema(list(arrow_return_type)))]
def raise_conversion_error(original_exception: Exception) -> None:
raise PySparkRuntimeError(
errorClass="UDTF_ARROW_DATA_CONVERSION_ERROR",
messageParameters={
"data": str(data),
"schema": self._udtf_desc.return_type.simpleString(),
"arrow_schema": str(arrow_return_type),
},
) from original_exception
try:
table = LocalDataToArrowConversion.convert(
data, self._udtf_desc.return_type, self._runner_conf.use_large_var_types
)
except PySparkValueError as e:
if e.getErrorClass() == "AXIS_LENGTH_MISMATCH":
raise PySparkRuntimeError(
errorClass="UDTF_RETURN_SCHEMA_MISMATCH",
messageParameters={
"expected": e.getMessageParameters()["expected_length"], # type: ignore[index]
"actual": e.getMessageParameters()["actual_length"], # type: ignore[index]
"func": method_name,
},
) from e
# Fall through to general conversion error
raise_conversion_error(e)
except Exception as e:
raise_conversion_error(e)
return self._verify_result(table, method_name).to_batches()
def _to_columns(self, batch: pa.RecordBatch) -> list:
# Convert each input column to a list of Python values per row.
return [
(
[conv(v) for v in ArrowTableToRowsConversion._to_pylist(column)]
if conv is not None
else ArrowTableToRowsConversion._to_pylist(column)
)
for column, conv in zip(batch.columns, self._converters)
]
def _convert_result(self, res: Any, method_name: str) -> Iterator[pa.RecordBatch]:
for batch in self._convert_rows_to_arrow(res, method_name):
yield ArrowBatchTransformer.wrap_struct(batch)