blob: 833ccd3ceb0a35c7b29b1ec9118ab1168b18720b [file]
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Type aliases and variables for the eval type handlers.
The Arrow element types are forward refs so pyarrow stays a type-checking-only
import at runtime, as elsewhere in the worker.
"""
from collections.abc import Iterator
from dataclasses import dataclass
from typing import TYPE_CHECKING, Any, Optional, Protocol, TypeVar
if TYPE_CHECKING:
import pyarrow as pa
from pyspark.sql.types import StructType
from pyspark.sql.udtf import AnalyzeResult
# Grouped-category input element, matching what the serializer yields: a group
# serializer yields a lazy iterator; a co-group serializer materializes each side.
GroupedBatch = Iterator["pa.RecordBatch"]
CoGroupedBatch = tuple[list["pa.RecordBatch"], list["pa.RecordBatch"]]
# Handler input and output stream element types.
InputBatch = TypeVar("InputBatch")
OutputBatch = TypeVar("OutputBatch")
@dataclass(frozen=True)
class UDF:
"""One UDF as the worker reads it; each handler takes the fields it needs."""
# The functions applied in turn to the UDF's arguments, as unpickled. EvalTypeHandler.__init__
# chains them into one as the handler's _raw_funcs, and keeps that wrapped for profiling and,
# for most eval types, in fail_on_stopiteration, as the handler's _funcs.
funcs: list[Any]
args_offsets: list[int]
kwargs_offsets: dict[str, int]
# The DataType of the last function in the chain; a StructType for the eval types that
# return rows.
return_type: Any
# The id the profiler reports the UDF's results under.
result_id: int
class UDTFInstance(Protocol):
"""An instance of the user's UDTF class, or the PartitionedUDTF wrapping one: an object whose
methods are called, not a function. ``terminate`` and ``cleanup`` are optional."""
def eval(self, *args: Any, **kwargs: Any) -> Any: ...
@dataclass(frozen=True)
class UDTF:
"""The UDTF as the worker reads it; ``UDTFHandlerBase.__init__`` instantiates it as the
handler's ``_udtf``."""
# The user's UDTF class, as unpickled.
udtf_cls: type
args_offsets: list[int]
kwargs_offsets: dict[str, int]
return_type: "StructType"
# The child indexes of the TABLE argument's PARTITION BY expressions, if any.
partition_child_indexes: list[int]
# Passed to the UDTF's __init__ when it takes one argument.
analyze_result: Optional["AnalyzeResult"]
# The UDTF's name in error messages.
name: str