| # | |
| # Licensed to the Apache Software Foundation (ASF) under one or more | |
| # contributor license agreements. See the NOTICE file distributed with | |
| # this work for additional information regarding copyright ownership. | |
| # The ASF licenses this file to You under the Apache License, Version 2.0 | |
| # (the "License"); you may not use this file except in compliance with | |
| # the License. You may obtain a copy of the License at | |
| # | |
| # http://www.apache.org/licenses/LICENSE-2.0 | |
| # | |
| # Unless required by applicable law or agreed to in writing, software | |
| # distributed under the License is distributed on an "AS IS" BASIS, | |
| # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| # See the License for the specific language governing permissions and | |
| # limitations under the License. | |
| # | |
| """Verification helpers for a UDF's result before it is written back to the JVM. | |
| Used by the eval type handlers. | |
| """ | |
| from collections.abc import Iterator | |
| from typing import TYPE_CHECKING, Any, Callable, Type, TypeVar, Union, cast, get_args, get_origin | |
| from pyspark.errors import PySparkRuntimeError, PySparkTypeError | |
| from pyspark.sql.types import DataType, StructType | |
| if TYPE_CHECKING: | |
| import pandas as pd | |
| import pyarrow as pa | |
| T = TypeVar("T") | |
| def _top_level_package(t: type) -> str: | |
| """Return the top-level package of ``t`` (``pandas`` for ``pd.DataFrame``).""" | |
| return (t.__module__ or "").split(".", 1)[0] | |
| def verify_iter_result_row_count( | |
| iterator: Iterator, | |
| expected_rows: Callable[[], int], | |
| ) -> Iterator: | |
| """Yield elements and verify final row count matches expected exactly. | |
| ``expected_rows`` is a callable because the expected count is only known once | |
| the iterator is fully consumed (input rows are counted lazily as a side effect | |
| of pulling batches), so it must be read after this generator is exhausted. | |
| """ | |
| actual_rows = 0 | |
| for element in iterator: | |
| actual_rows += len(element) | |
| yield element | |
| verify_result_row_count(actual_rows, expected_rows()) | |
| def verify_iterator_exhausted(iterator: Iterator) -> None: | |
| """Verify that an iterator has been fully consumed.""" | |
| try: | |
| next(iterator) | |
| except StopIteration: | |
| pass | |
| else: | |
| raise PySparkRuntimeError(errorClass="INPUT_NOT_FULLY_CONSUMED", messageParameters={}) | |
| def verify_output_row_limit( | |
| iterator: Iterator, | |
| max_rows: Union[int, Callable[[], int]], | |
| ) -> Iterator: | |
| """Yield elements while verifying total rows do not exceed a limit (fail-fast).""" | |
| total_rows = 0 | |
| for element in iterator: | |
| total_rows += len(element) | |
| if total_rows > (max_rows() if callable(max_rows) else max_rows): | |
| raise PySparkRuntimeError(errorClass="OUTPUT_EXCEEDS_INPUT_ROWS", messageParameters={}) | |
| yield element | |
| def verify_result_row_count(result_length: int, expected: int) -> None: | |
| """Raise if the result row count doesn't match the expected input row count.""" | |
| if result_length != expected: | |
| raise PySparkRuntimeError( | |
| errorClass="RESULT_ROWS_MISMATCH", | |
| messageParameters={ | |
| "output_length": str(result_length), | |
| "input_length": str(expected), | |
| }, | |
| ) | |
| def verify_return_type(result: T, expected_type: Type[T]) -> T: | |
| """ | |
| Verify a UDF return value against an expected type. | |
| Returns ``result`` unchanged if ``isinstance(result, expected_type)``. | |
| For ``Iterator[T]``, returns a lazy iterator that checks each element | |
| against ``T`` on consumption. Raises ``PySparkTypeError`` on mismatch. | |
| """ | |
| if get_origin(expected_type) is Iterator: | |
| (element_type,) = get_args(expected_type) | |
| label = f"iterator of {_top_level_package(element_type)}.{element_type.__name__}" | |
| if not isinstance(result, Iterator): | |
| raise PySparkTypeError( | |
| errorClass="UDF_RETURN_TYPE", | |
| messageParameters={"expected": label, "actual": type(result).__name__}, | |
| ) | |
| def check_element(element: T) -> T: | |
| if not isinstance(element, element_type): | |
| raise PySparkTypeError( | |
| errorClass="UDF_RETURN_TYPE", | |
| messageParameters={ | |
| "expected": label, | |
| "actual": f"iterator of {type(element).__name__}", | |
| }, | |
| ) | |
| return element | |
| return cast(T, map(check_element, result)) | |
| if not isinstance(result, expected_type): | |
| raise PySparkTypeError( | |
| errorClass="UDF_RETURN_TYPE", | |
| messageParameters={ | |
| "expected": f"{_top_level_package(expected_type)}.{expected_type.__name__}", | |
| "actual": type(result).__name__, | |
| }, | |
| ) | |
| return result | |
| def verify_scalar_result(result: Any, num_rows: int) -> Any: | |
| """ | |
| Verify a scalar UDF result is array-like and has the expected number of rows. | |
| Parameters | |
| ---------- | |
| result : Any | |
| The UDF result to verify. | |
| num_rows : int | |
| Expected number of rows (must match input batch size). | |
| """ | |
| try: | |
| result_length = len(result) | |
| except TypeError: | |
| raise PySparkTypeError( | |
| errorClass="UDF_RETURN_TYPE", | |
| messageParameters={ | |
| "expected": "array-like object", | |
| "actual": type(result).__name__, | |
| }, | |
| ) | |
| verify_result_row_count(result_length, num_rows) | |
| return result | |
| def _verify_column_schema( | |
| actual_names: list, expected_names: list, *, assign_cols_by_name: bool | |
| ) -> None: | |
| """Check column names (by-name) or count (by-position) match the expected schema.""" | |
| if assign_cols_by_name: | |
| actual_set = set(actual_names) | |
| expected_set = set(expected_names) | |
| missing = sorted(expected_set.difference(actual_set)) | |
| extra = sorted(actual_set.difference(expected_set)) | |
| if missing or extra: | |
| raise PySparkRuntimeError( | |
| errorClass="RESULT_COLUMN_NAMES_MISMATCH", | |
| messageParameters={ | |
| "missing": f" Missing: {', '.join(missing)}." if missing else "", | |
| "extra": f" Unexpected: {', '.join(extra)}." if extra else "", | |
| }, | |
| ) | |
| elif len(actual_names) != len(expected_names): | |
| raise PySparkRuntimeError( | |
| errorClass="RESULT_COLUMN_SCHEMA_MISMATCH", | |
| messageParameters={ | |
| "expected": str(len(expected_names)), | |
| "actual": str(len(actual_names)), | |
| }, | |
| ) | |
| def verify_pandas_result( | |
| result: Union["pd.DataFrame", "pd.Series"], | |
| return_type: DataType, | |
| assign_cols_by_name: bool, | |
| truncate_return_schema: bool, | |
| ) -> None: | |
| import pandas as pd | |
| if not isinstance(return_type, StructType): | |
| verify_return_type(result, pd.Series) | |
| return | |
| verify_return_type(result, pd.DataFrame) | |
| # Skip schema check on a fully empty result (no rows and no columns). | |
| if result.empty and len(result.columns) == 0: | |
| return | |
| field_names = [field.name for field in return_type.fields] | |
| actual_names = ( | |
| list(result.columns[: len(field_names)]) if truncate_return_schema else list(result.columns) | |
| ) | |
| # By-name mode only applies when the result has string column names; | |
| # a numeric RangeIndex falls back to a by-position count check. | |
| by_name = assign_cols_by_name and any(isinstance(n, str) for n in result.columns) | |
| _verify_column_schema(actual_names, field_names, assign_cols_by_name=by_name) | |
| def verify_arrow_result( | |
| result: Union["pa.Table", "pa.RecordBatch"], | |
| assign_cols_by_name: bool, | |
| expected_cols_and_types: Union[dict[str, "pa.DataType"], list[tuple[str, "pa.DataType"]]], | |
| ) -> None: | |
| # Skip schema check on a fully empty result (no rows and no columns). | |
| if result.num_columns == 0 and result.num_rows == 0: | |
| return | |
| actual_names = list(result.schema.names) | |
| actual_types = list(result.schema.types) | |
| # expected_cols_and_types is a dict in by-name mode, list of (name, type) by position. | |
| if isinstance(expected_cols_and_types, dict): | |
| expected_names = list(expected_cols_and_types.keys()) | |
| else: | |
| expected_names = [name for name, _ in expected_cols_and_types] | |
| _verify_column_schema(actual_names, expected_names, assign_cols_by_name=assign_cols_by_name) | |
| if isinstance(expected_cols_and_types, dict): | |
| actual_by_name = dict(zip(actual_names, actual_types)) | |
| column_types = [ | |
| (name, expected_cols_and_types[name], actual_by_name[name]) | |
| for name in sorted(expected_cols_and_types.keys()) | |
| ] | |
| else: | |
| column_types = [ | |
| (expected_name, expected_type, actual_type) | |
| for (expected_name, expected_type), actual_type in zip( | |
| expected_cols_and_types, actual_types | |
| ) | |
| ] | |
| type_mismatch = [ | |
| (name, expected, actual) for name, expected, actual in column_types if actual != expected | |
| ] | |
| if type_mismatch: | |
| raise PySparkRuntimeError( | |
| errorClass="RESULT_COLUMN_TYPES_MISMATCH", | |
| messageParameters={ | |
| "mismatch": ", ".join( | |
| "column '{}' (expected {}, actual {})".format(name, expected, actual) | |
| for name, expected, actual in type_mismatch | |
| ) | |
| }, | |
| ) |