blob: 56503ba9eef037db9fb5977038515abea63af964 [file]
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
"""Handlers for the stateful streaming UDF eval types (``applyInPandasWithState``,
``transformWithStateInPandas`` and ``transformWithState`` over ``Row``).
The input stream is a flat ``Iterator[pa.RecordBatch]`` in which the rows of one grouping
key arrive sequentially, possibly spanning batch boundaries, so each handler regroups the
rows itself rather than relying on a grouped serializer.
pandas and pyarrow are imported lazily (inside ``run`` and the type-checking block) so the
module stays importable and its handlers register without either installed. Each handler
calls the matching ``require_minimum_*_version`` in ``__init__`` so a missing or too-old
dependency surfaces a clear error when the handler runs.
"""
from __future__ import annotations
import itertools
import json
from collections.abc import Iterable, Iterator
from typing import TYPE_CHECKING, Any, Optional
from pyspark.errors import PySparkRuntimeError, PySparkTypeError, PySparkValueError
from pyspark.eval_handlers._base import BatchEvalTypeHandler
from pyspark.eval_handlers.utils import extract_key_value_indexes
from pyspark.serializers import CPickleSerializer
from pyspark.sql.conversion import (
ArrowBatchTransformer,
ArrowToPandasConversion,
PandasToArrowConversion,
)
from pyspark.sql.pandas.types import to_arrow_type
from pyspark.sql.pandas.utils import (
require_minimum_pandas_version,
require_minimum_pyarrow_version,
)
from pyspark.sql.streaming.state import GroupState
from pyspark.sql.streaming.stateful_processor_api_client import StatefulProcessorApiClient
from pyspark.sql.streaming.stateful_processor_util import TransformWithStateInPandasFuncMode
from pyspark.sql.types import (
BinaryType,
IntegerType,
LongType,
Row,
StringType,
StructField,
StructType,
)
from pyspark.util import PythonEvalType
if TYPE_CHECKING:
import pandas as pd
import pyarrow as pa
from pyspark.eval_handlers._typing import UDF
from pyspark.worker_util import EvalConf, RunnerConf
class PandasGroupedMapWithStateUDFHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
"""SQL_GROUPED_MAP_PANDAS_UDF_WITH_STATE (``applyInPandasWithState``): regroup the input
chunks by grouping key, invoke the UDF once per key with a lazy iterator of pandas
DataFrames and its ``GroupState``, and batch the output DataFrames plus the updated state
into count/data/state RecordBatches."""
eval_type = PythonEvalType.SQL_GROUPED_MAP_PANDAS_UDF_WITH_STATE
_iter_based = True
# The output RecordBatch has three struct fields, accessed by position
# (_0/_1/_2, not by name): a count column indicating how many data and
# state rows are present, the UDF output data, and the serialized state.
_RESULT_COUNT_DF_TYPE = StructType(
[
StructField("dataCount", IntegerType()),
StructField("stateCount", IntegerType()),
]
)
_RESULT_STATE_DF_TYPE = StructType(
[
StructField("properties", StringType()),
StructField("keyRowAsUnsafe", BinaryType()),
StructField("object", BinaryType()),
StructField("oldTimeoutTimestamp", LongType()),
]
)
def __init__(self, udf_descs: list[UDF], runner_conf: RunnerConf, eval_conf: EvalConf) -> None:
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
# See FlatMapGroupsInPandasWithStateExec for how arg_offsets are used to
# distinguish between grouping attributes and data attributes.
parsed_offsets = extract_key_value_indexes(self._udf_desc.args_offsets)
self._key_offsets = parsed_offsets[0][0]
self._value_offsets = parsed_offsets[0][1]
self._state_object_schema = eval_conf.state_value_schema
arrow_max_records_per_batch = runner_conf.arrow_max_records_per_batch
self._arrow_max_records_per_batch = (
arrow_max_records_per_batch if arrow_max_records_per_batch > 0 else 2**31 - 1
)
self._pickle_ser = CPickleSerializer()
def run(self, split_index: int, data: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]:
"""Apply applyInPandasWithState UDF.
The input batches carry the data columns plus a trailing state-info
column. Chunks are regrouped by grouping key (via the reused state
object), the UDF is invoked once per key with a lazy iterator of
data DataFrames and its state, and the output DataFrames plus updated
state are batched into count/data/state RecordBatches bounded by
arrow_max_records_per_batch.
"""
import pandas as pd
import pyarrow as pa
def to_pandas(batch: pa.RecordBatch) -> list:
return ArrowToPandasConversion.to_pandas(
batch,
timezone=self._runner_conf.timezone,
struct_in_pandas="dict",
ndarray_as_list=False,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
df_for_struct=False,
)
def construct_state(state_info_col: dict) -> GroupState:
"""Construct a state instance from the value of the state info column."""
state_properties = json.loads(state_info_col["properties"])
state_info_col_object = state_info_col["object"]
if state_info_col_object:
state_object = self._pickle_ser.loads(state_info_col_object)
else:
state_object = None
state_properties["optionalValue"] = state_object
return GroupState(
keyAsUnsafe=state_info_col["keyRowAsUnsafe"],
valueSchema=self._state_object_schema, # type: ignore[arg-type]
**state_properties,
)
def gen_data_and_state(
batches: Iterator[pa.RecordBatch],
) -> Iterator[tuple]:
"""Deserialize ArrowRecordBatches into (list of pandas.Series, state) chunks.
Each batch carries the data columns plus a trailing state-info
column. For every state-info row, the matching data slice is cut
out via its (startOffset, numRows) and converted to pandas. A single
state instance is reused across all chunks of one grouping key (the
key appears sequentially and its last chunk is flagged), so grouping
the output by the state object is equivalent to grouping by key.
This must not materialize multiple Arrow RecordBatches at once.
"""
state_for_current_group = None
for batch in batches:
batch_schema = batch.schema
data_schema = pa.schema([batch_schema[i] for i in range(0, len(batch_schema) - 1)])
state_schema = pa.schema([batch_schema[-1]])
batch_columns = batch.columns
data_columns = batch_columns[0:-1]
state_column = batch_columns[-1]
data_batch = pa.RecordBatch.from_arrays(data_columns, schema=data_schema)
state_batch = pa.RecordBatch.from_arrays([state_column], schema=state_schema)
state_pandas = to_pandas(state_batch)[0]
for state_idx in range(0, len(state_pandas)):
state_info_col = state_pandas.iloc[state_idx]
if not state_info_col:
# no more data with grouping key + state
break
data_start_offset = state_info_col["startOffset"]
num_data_rows = state_info_col["numRows"]
is_last_chunk = state_info_col["isLastChunk"]
if state_for_current_group:
# reuse the state already built for this group
state = state_for_current_group
else:
# first occurrence of this group, construct a new state
state = construct_state(state_info_col)
if is_last_chunk:
# last chunk for this group, drop the cached state
state_for_current_group = None
elif not state_for_current_group:
# more chunks expected for this group, cache the state
state_for_current_group = state
data_batch_for_group = data_batch.slice(data_start_offset, num_data_rows)
yield (to_pandas(data_batch_for_group), state)
def verify_element(result: pd.DataFrame) -> pd.DataFrame:
if not isinstance(result, pd.DataFrame):
raise PySparkTypeError(
errorClass="UDF_RETURN_TYPE",
messageParameters={
"expected": "iterator of pandas.DataFrame",
"actual": "iterator of {}".format(type(result).__name__),
},
)
# The number of columns of the result must match the return type,
# but an empty result with no columns at all is acceptable.
if not (
len(result.columns) == len(self._udf_desc.return_type)
or (len(result.columns) == 0 and result.empty)
):
raise PySparkRuntimeError(
errorClass="RESULT_COLUMN_SCHEMA_MISMATCH",
messageParameters={
"expected": str(len(self._udf_desc.return_type)),
"actual": str(len(result.columns)),
},
)
return result
def apply_udf_to_group(
key_series: list, value_series_gen: Iterator[list], state: GroupState
) -> Iterator[pd.DataFrame]:
"""Adapt the deserialized chunks to the user function signature.
Extract the scalar grouping key, convert each chunk of value Series
into a pandas DataFrame (lazily), invoke the UDF, and validate that
every returned element is a pandas.DataFrame conforming to the
return type.
"""
key = tuple(s[0] for s in key_series)
values: Iterable
if state.hasTimedOut:
# On timeout the UDF is called with an empty DataFrame instead
# of an empty iterator.
values = [
pd.DataFrame(columns=pd.concat(next(value_series_gen), axis=1).columns),
]
else:
values = (pd.concat(x, axis=1) for x in value_series_gen)
result_iter = self._func(key, values, state)
if isinstance(result_iter, pd.DataFrame):
raise PySparkTypeError(
errorClass="UDF_RETURN_TYPE",
messageParameters={
"expected": "iterable of pandas.DataFrame",
"actual": type(result_iter).__name__,
},
)
try:
iter(result_iter)
except TypeError:
raise PySparkTypeError(
errorClass="UDF_RETURN_TYPE",
messageParameters={
"expected": "iterable",
"actual": type(result_iter).__name__,
},
)
return (verify_element(x) for x in result_iter)
def construct_state_pdf(state: GroupState) -> pd.DataFrame:
"""Construct a single-row pandas DataFrame from the state instance."""
state_properties = state.json().encode("utf-8")
state_key_row_as_binary = state._keyAsUnsafe
if state.exists:
state_object = self._pickle_ser.dumps(state._value_schema.toInternal(state._value))
else:
state_object = None
state_old_timeout_timestamp = state.oldTimeoutTimestamp
state_dict = {
"properties": [state_properties],
"keyRowAsUnsafe": [state_key_row_as_binary],
"object": [state_object],
"oldTimeoutTimestamp": [state_old_timeout_timestamp],
}
return pd.DataFrame.from_dict(state_dict)
def construct_record_batch(
pdfs: list,
pdf_data_cnt: int,
pdf_schema: StructType,
state_pdfs: list,
state_data_cnt: int,
) -> pa.RecordBatch:
"""Construct a count/data/state RecordBatch from output DataFrames and states.
Arrow RecordBatch requires all columns to have the same number of
rows, so data and state are padded with empty rows to the max of the
two counts; the count column records the real (unpadded) sizes.
"""
max_data_cnt = max(1, max(pdf_data_cnt, state_data_cnt))
# Only the first row of the count column is meaningful; the rest
# repeat the same values for friendlier compression.
count_dict = {
"dataCount": [pdf_data_cnt] * max_data_cnt,
"stateCount": [state_data_cnt] * max_data_cnt,
}
count_pdf = pd.DataFrame.from_dict(count_dict)
empty_row_cnt_in_data = max_data_cnt - pdf_data_cnt
empty_row_cnt_in_state = max_data_cnt - state_data_cnt
empty_rows_pdf = pd.DataFrame(
dict.fromkeys(pdf_schema.names),
index=[x for x in range(0, empty_row_cnt_in_data)],
)
empty_rows_state = pd.DataFrame(
columns=["properties", "keyRowAsUnsafe", "object", "oldTimeoutTimestamp"],
index=[x for x in range(0, empty_row_cnt_in_state)],
)
pdfs.append(empty_rows_pdf)
state_pdfs.append(empty_rows_state)
merged_pdf = pd.concat(pdfs, ignore_index=True)
merged_state_pdf = pd.concat(state_pdfs, ignore_index=True)
# Fields map to _0=count, _1=output data, _2=state data.
data = [count_pdf, merged_pdf, merged_state_pdf]
schema = StructType(
[
StructField("_0", self._RESULT_COUNT_DF_TYPE),
StructField("_1", pdf_schema),
StructField("_2", self._RESULT_STATE_DF_TYPE),
]
)
return PandasToArrowConversion.from_pandas(
data,
schema,
timezone=self._runner_conf.timezone,
safecheck=self._runner_conf.safecheck,
arrow_cast=True,
assign_cols_by_name=self._runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=self._runner_conf.int_to_decimal_coercion_enabled,
)
def result_and_state_stream() -> Iterator[tuple]:
# The same state object is reused across all chunks of a group,
# so grouping by it is equivalent to grouping by key.
for state, group in itertools.groupby(gen_data_and_state(data), key=lambda x: x[1]):
# These must stay lazy - do not materialize the data chunks.
data_gen = (data_pandas for data_pandas, _ in group)
# Consume the first chunk to extract the grouping key series.
first_elem = next(data_gen)
key_series = [first_elem[o] for o in self._key_offsets]
value_series_gen = (
[x[o] for o in self._value_offsets]
for x in itertools.chain([first_elem], data_gen)
)
yield (apply_udf_to_group(key_series, value_series_gen, state), state)
pdfs: list = []
state_pdfs: list = []
pdf_data_cnt = 0
state_data_cnt = 0
for result_iter, state in result_and_state_stream():
for pdf in result_iter:
# Ignore empty pandas DataFrames.
if len(pdf) > 0:
pdf_data_cnt += len(pdf)
pdfs.append(pdf)
# Flush a batch once the record threshold is exceeded.
if pdf_data_cnt > self._arrow_max_records_per_batch:
yield construct_record_batch(
pdfs,
pdf_data_cnt,
self._udf_desc.return_type,
state_pdfs,
state_data_cnt,
)
pdfs = []
state_pdfs = []
pdf_data_cnt = 0
state_data_cnt = 0
# The state must be captured after the result iterator is fully
# consumed, so the UDF has run and the state is up to date.
state_pdfs.append(construct_state_pdf(state))
state_data_cnt += 1
# Flush the trailing batch if it has any data or state left.
if pdf_data_cnt > 0 or state_data_cnt > 0:
yield construct_record_batch(
pdfs, pdf_data_cnt, self._udf_desc.return_type, state_pdfs, state_data_cnt
)
class PandasTransformWithStateUDFHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
"""SQL_TRANSFORM_WITH_STATE_PANDAS_UDF (``transformWithStateInPandas``): regroup the input
rows by grouping key into bounded pandas DataFrame chunks, invoke the UDF once per key
(PROCESS_DATA), then once for PROCESS_TIMER and once for COMPLETE, talking to the JVM state
server through a ``StatefulProcessorApiClient``."""
eval_type = PythonEvalType.SQL_TRANSFORM_WITH_STATE_PANDAS_UDF
def __init__(self, udf_descs: list[UDF], runner_conf: RunnerConf, eval_conf: EvalConf) -> None:
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
# See TransformWithStateInPandasExec for how arg_offsets are used to
# distinguish between grouping attributes and data attributes
parsed_offsets = extract_key_value_indexes(self._udf_desc.args_offsets)
assert len(parsed_offsets) == 1, (
"Expected one pair of offsets for TRANSFORM_WITH_STATE_PANDAS UDF."
)
self._key_offsets = parsed_offsets[0][0]
self._value_offsets = parsed_offsets[0][1]
self._output_schema = StructType([StructField("_0", self._udf_desc.return_type)])
self._stateful_processor_api_client = StatefulProcessorApiClient(
eval_conf.state_server_socket_port, # type: ignore[arg-type]
eval_conf.grouping_key_schema, # type: ignore[arg-type]
auth_secret=eval_conf.state_server_auth_secret,
)
arrow_max_records_per_batch = runner_conf.arrow_max_records_per_batch
self._arrow_max_records_per_batch = (
arrow_max_records_per_batch if arrow_max_records_per_batch > 0 else 2**31 - 1
)
self._arrow_max_bytes_per_batch = runner_conf.arrow_max_bytes_per_batch
def run(self, split_index: int, data: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]:
"""Apply transformWithStateInPandas UDF.
Data chunks for the same grouping key appear sequentially in the
input batches but may span batch boundaries, so rows are regrouped
by key and re-chunked into pandas DataFrames bounded by
arrow_max_records_per_batch and arrow_max_bytes_per_batch. The UDF
is invoked once per grouping key with a lazy iterator of chunks,
then once for PROCESS_TIMER and once for COMPLETE.
"""
import pandas as pd
total_bytes = 0
total_rows = 0
average_arrow_row_size = 0.0
def row_stream() -> Iterator[tuple]:
nonlocal total_bytes, total_rows, average_arrow_row_size
for batch in data:
# Short circuit batch size stats if the batch size is
# unlimited as computing batch size is computationally
# expensive.
if self._arrow_max_bytes_per_batch != 2**31 - 1 and batch.num_rows > 0:
total_bytes += sum(
buf.size
for col in batch.columns
for buf in col.buffers()
if buf is not None
)
total_rows += batch.num_rows
average_arrow_row_size = total_bytes / total_rows
data_pandas = ArrowToPandasConversion.to_pandas(
batch,
timezone=self._runner_conf.timezone,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
)
for row in pd.concat(data_pandas, axis=1).itertuples(index=False):
batch_key = tuple(row[o] for o in self._key_offsets)
yield (batch_key, row)
def generate_data_batches() -> Iterator[tuple]:
"""
Deserialize ArrowRecordBatches and return a generator of
(grouping key, pandas.DataFrame) chunks.
This function must avoid materializing multiple Arrow
RecordBatches into memory at the same time, and data chunks
from the same grouping key should appear sequentially.
"""
for batch_key, group_rows in itertools.groupby(row_stream(), key=lambda x: x[0]):
rows = []
for _, row in group_rows:
rows.append(row)
if (
len(rows) >= self._arrow_max_records_per_batch
or len(rows) * average_arrow_row_size >= self._arrow_max_bytes_per_batch
):
yield (batch_key, pd.DataFrame(rows))
rows = []
if rows:
yield (batch_key, pd.DataFrame(rows))
def convert_results(result_iter: Iterable[pd.DataFrame]) -> Iterator[pa.RecordBatch]:
for result in result_iter:
if isinstance(self._udf_desc.return_type, StructType) and not isinstance(
result, pd.DataFrame
):
raise PySparkValueError(
"Invalid return type. Please make sure that the UDF returns a "
"pandas.DataFrame when the specified return type is StructType."
)
yield PandasToArrowConversion.from_pandas(
[result],
self._output_schema,
timezone=self._runner_conf.timezone,
safecheck=self._runner_conf.safecheck,
arrow_cast=True,
assign_cols_by_name=self._runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=(
self._runner_conf.int_to_decimal_coercion_enabled
),
)
for key, group in itertools.groupby(generate_data_batches(), key=lambda x: x[0]):
# This must be a generator expression - do not materialize.
values_gen = (df.iloc[:, self._value_offsets] for _, df in group)
yield from convert_results(
self._func(
self._stateful_processor_api_client,
TransformWithStateInPandasFuncMode.PROCESS_DATA,
key,
values_gen,
)
)
yield from convert_results(
self._func(
self._stateful_processor_api_client,
TransformWithStateInPandasFuncMode.PROCESS_TIMER,
None,
iter([]),
)
)
yield from convert_results(
self._func(
self._stateful_processor_api_client,
TransformWithStateInPandasFuncMode.COMPLETE,
None,
iter([]),
)
)
class PandasTransformWithStateInitStateUDFHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
"""SQL_TRANSFORM_WITH_STATE_PANDAS_INIT_STATE_UDF (``transformWithStateInPandas`` with an
initial state): like ``PandasTransformWithStateUDFHandler``, but each input batch carries
either input data or initial state, and the UDF receives the two as separate lazy
iterators of pandas DataFrames."""
eval_type = PythonEvalType.SQL_TRANSFORM_WITH_STATE_PANDAS_INIT_STATE_UDF
def __init__(self, udf_descs: list[UDF], runner_conf: RunnerConf, eval_conf: EvalConf) -> None:
require_minimum_pandas_version()
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
# See TransformWithStateInPandasExec for how arg_offsets are used to
# distinguish between grouping attributes and data attributes.
# parsed offsets:
# [
# [groupingKeyOffsets, dedupDataOffsets],
# [initStateGroupingOffsets, dedupInitDataOffsets]
# ]
parsed_offsets = extract_key_value_indexes(self._udf_desc.args_offsets)
self._key_offsets = parsed_offsets[0][0]
self._init_key_offsets = parsed_offsets[1][0]
self._output_schema = StructType([StructField("_0", self._udf_desc.return_type)])
self._stateful_processor_api_client = StatefulProcessorApiClient(
eval_conf.state_server_socket_port, # type: ignore[arg-type]
eval_conf.grouping_key_schema, # type: ignore[arg-type]
auth_secret=eval_conf.state_server_auth_secret,
)
arrow_max_records_per_batch = runner_conf.arrow_max_records_per_batch
self._arrow_max_records_per_batch = (
arrow_max_records_per_batch if arrow_max_records_per_batch > 0 else 2**31 - 1
)
self._arrow_max_bytes_per_batch = runner_conf.arrow_max_bytes_per_batch
def run(self, split_index: int, data: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]:
"""Apply transformWithStateInPandas UDF with initial state.
The input batches carry two struct columns, ``inputData`` and
``initState``; each batch holds one or the other but never both.
Rows are flattened out of whichever struct is present, regrouped by
grouping key, and re-chunked into pandas DataFrames bounded by
arrow_max_records_per_batch and arrow_max_bytes_per_batch. The UDF
is invoked once per grouping key with two separate lazy iterators
(data DataFrames and init-state DataFrames), then once for
PROCESS_TIMER and once for COMPLETE.
"""
import pandas as pd
import pyarrow as pa
total_bytes = 0
total_rows = 0
average_arrow_row_size = 0.0
def flatten_columns(cur_batch: pa.RecordBatch, col_name: str) -> pa.Table:
struct_column = cur_batch.column(cur_batch.schema.get_field_index(col_name))
# Check if the entire column is null: an empty table (no columns)
# signals the struct is absent from this batch.
if struct_column.null_count == len(struct_column):
return pa.Table.from_arrays([], names=[])
field_names = [struct_column.type[i].name for i in range(struct_column.type.num_fields)]
field_arrays = [struct_column.field(i) for i in range(struct_column.type.num_fields)]
return pa.Table.from_arrays(field_arrays, names=field_names)
def to_pandas(table: pa.Table) -> list:
return ArrowToPandasConversion.to_pandas(
table,
timezone=self._runner_conf.timezone,
prefer_int_ext_dtype=self._runner_conf.prefer_int_ext_dtype,
)
def row_stream() -> Iterator[tuple]:
nonlocal total_bytes, total_rows, average_arrow_row_size
for batch in data:
# Short circuit batch size stats if the batch size is
# unlimited as computing batch size is computationally
# expensive.
if self._arrow_max_bytes_per_batch != 2**31 - 1 and batch.num_rows > 0:
total_bytes += sum(
buf.size
for col in batch.columns
for buf in col.buffers()
if buf is not None
)
total_rows += batch.num_rows
average_arrow_row_size = total_bytes / total_rows
data_table = flatten_columns(batch, "inputData")
init_table = flatten_columns(batch, "initState")
# Empty table has no columns. Each batch carries either
# input data or init state, never both.
has_data = data_table.num_columns > 0
has_init = init_table.num_columns > 0
assert not (has_data and has_init)
if has_data:
for row in pd.concat(to_pandas(data_table), axis=1).itertuples(index=False):
batch_key = tuple(row[o] for o in self._key_offsets)
yield (batch_key, row, None)
elif has_init:
for row in pd.concat(to_pandas(init_table), axis=1).itertuples(index=False):
batch_key = tuple(row[o] for o in self._init_key_offsets)
yield (batch_key, None, row)
empty_dataframe = pd.DataFrame()
def generate_data_batches() -> Iterator[tuple]:
"""
Deserialize ArrowRecordBatches and return a generator of
(grouping key, data DataFrame, init-state DataFrame) chunks.
This function must avoid materializing multiple Arrow
RecordBatches into memory at the same time, and data chunks
from the same grouping key should appear sequentially.
"""
for batch_key, group_rows in itertools.groupby(row_stream(), key=lambda x: x[0]):
rows = []
init_state_rows = []
for _, row, init_state_row in group_rows:
if row is not None:
rows.append(row)
if init_state_row is not None:
init_state_rows.append(init_state_row)
total_len = len(rows) + len(init_state_rows)
if (
total_len >= self._arrow_max_records_per_batch
or total_len * average_arrow_row_size >= self._arrow_max_bytes_per_batch
):
yield (
batch_key,
pd.DataFrame(rows) if rows else empty_dataframe.copy(),
(
pd.DataFrame(init_state_rows)
if init_state_rows
else empty_dataframe.copy()
),
)
rows = []
init_state_rows = []
if rows or init_state_rows:
yield (
batch_key,
pd.DataFrame(rows) if rows else empty_dataframe.copy(),
(
pd.DataFrame(init_state_rows)
if init_state_rows
else empty_dataframe.copy()
),
)
def convert_results(
result_iter: Iterable[pd.DataFrame],
) -> Iterator[pa.RecordBatch]:
# TODO(SPARK-49100): add verification that elements in result_iter are
# indeed of type pd.DataFrame and conform to assigned cols
for result in result_iter:
if isinstance(self._udf_desc.return_type, StructType) and not isinstance(
result, pd.DataFrame
):
raise PySparkValueError(
"Invalid return type. Please make sure that the UDF returns a "
"pandas.DataFrame when the specified return type is StructType."
)
yield PandasToArrowConversion.from_pandas(
[result],
self._output_schema,
timezone=self._runner_conf.timezone,
safecheck=self._runner_conf.safecheck,
arrow_cast=True,
assign_cols_by_name=self._runner_conf.assign_cols_by_name,
int_to_decimal_coercion_enabled=(
self._runner_conf.int_to_decimal_coercion_enabled
),
)
for key, group in itertools.groupby(generate_data_batches(), key=lambda x: x[0]):
# These must be generator expressions - do not materialize. The
# UDF receives the data and init-state DataFrames as two
# separate iterators, with empty chunks filtered out.
group_data, group_init = itertools.tee(group, 2)
state_values = (data_df for _, data_df, _ in group_data if not data_df.empty)
init_states = (init_df for _, _, init_df in group_init if not init_df.empty)
yield from convert_results(
self._func(
self._stateful_processor_api_client,
TransformWithStateInPandasFuncMode.PROCESS_DATA,
key,
state_values,
init_states,
)
)
yield from convert_results(
self._func(
self._stateful_processor_api_client,
TransformWithStateInPandasFuncMode.PROCESS_TIMER,
None,
iter([]),
iter([]),
)
)
yield from convert_results(
self._func(
self._stateful_processor_api_client,
TransformWithStateInPandasFuncMode.COMPLETE,
None,
iter([]),
iter([]),
)
)
class RowTransformWithStateUDFHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
"""SQL_TRANSFORM_WITH_STATE_PYTHON_ROW_UDF (``transformWithState`` over ``Row``): regroup the
input rows by grouping key, invoke the UDF once per key (PROCESS_DATA) with a lazy iterator
of Rows, then once for PROCESS_TIMER and once for COMPLETE, talking to the JVM state server
through a ``StatefulProcessorApiClient``."""
eval_type = PythonEvalType.SQL_TRANSFORM_WITH_STATE_PYTHON_ROW_UDF
def __init__(self, udf_descs: list[UDF], runner_conf: RunnerConf, eval_conf: EvalConf) -> None:
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
# See TransformWithStateInPySparkExec for how arg_offsets are used to
# distinguish between grouping attributes and data attributes.
parsed_offsets = extract_key_value_indexes(self._udf_desc.args_offsets)
self._key_offsets = parsed_offsets[0][0]
self._stateful_processor_api_client = StatefulProcessorApiClient(
eval_conf.state_server_socket_port, # type: ignore[arg-type]
eval_conf.grouping_key_schema, # type: ignore[arg-type]
auth_secret=eval_conf.state_server_auth_secret,
)
def run(self, split_index: int, data: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]:
"""Apply transformWithStateInPySpark UDF over Row objects.
Input batches are read row by row without materializing whole
batches. Rows carrying the same grouping key appear sequentially, so
they are regrouped by key and the UDF is invoked once per key with a
lazy iterator of Row objects, then once for PROCESS_TIMER and once
for COMPLETE. The UDF yields (iterator of Row, Spark type) pairs that
are converted back into Arrow RecordBatches wrapped in a single
struct column for the output stream.
"""
import pyarrow as pa
def generate_data_batches() -> Iterator[tuple[Any, Any]]:
"""
Deserialize ArrowRecordBatches and return a generator of
(grouping key, Row) tuples.
This function must avoid materializing multiple Arrow
RecordBatches into memory at the same time, and data chunks
from the same grouping key should appear sequentially.
"""
for batch in data:
DataRow = Row(*batch.schema.names)
# Iterate row by row without converting the whole batch.
num_cols = batch.num_columns
for row_idx in range(batch.num_rows):
row_key = tuple(batch[o][row_idx].as_py() for o in self._key_offsets)
row = DataRow(*(batch.column(i)[row_idx].as_py() for i in range(num_cols)))
yield row_key, row
def convert_results(result_rows: Iterable[Any]) -> Iterator[pa.RecordBatch]:
# TODO(SPARK-XXXXX): add verification that elements in result_rows
# are indeed of type Row and conform to assigned cols
# Convert spark type to arrow type
# TODO: we need to make this configurable, currently using default values.
arrow_type = to_arrow_type(
self._udf_desc.return_type,
timezone="UTC",
prefers_large_types=False,
)
rows_as_dict = [row.asDict(True) for row in result_rows]
pdf_schema = pa.schema(list(arrow_type))
record_batch = pa.RecordBatch.from_pylist(rows_as_dict, schema=pdf_schema)
yield ArrowBatchTransformer.wrap_struct(record_batch)
for key, group in itertools.groupby(generate_data_batches(), key=lambda x: x[0]):
# This must be a generator expression - do not materialize.
values_gen = map(lambda x: x[1], group)
yield from convert_results(
self._func(
self._stateful_processor_api_client,
TransformWithStateInPandasFuncMode.PROCESS_DATA,
key,
values_gen,
)
)
yield from convert_results(
self._func(
self._stateful_processor_api_client,
TransformWithStateInPandasFuncMode.PROCESS_TIMER,
None,
iter([]),
)
)
yield from convert_results(
self._func(
self._stateful_processor_api_client,
TransformWithStateInPandasFuncMode.COMPLETE,
None,
iter([]),
)
)
class RowTransformWithStateInitStateUDFHandler(BatchEvalTypeHandler["pa.RecordBatch"]):
"""SQL_TRANSFORM_WITH_STATE_PYTHON_ROW_INIT_STATE_UDF (``transformWithState`` over ``Row``
with an initial state): like ``RowTransformWithStateUDFHandler``, but each input batch
carries either input data or initial state, and the UDF receives the two as separate
iterators of Rows per chunk."""
eval_type = PythonEvalType.SQL_TRANSFORM_WITH_STATE_PYTHON_ROW_INIT_STATE_UDF
def __init__(self, udf_descs: list[UDF], runner_conf: RunnerConf, eval_conf: EvalConf) -> None:
require_minimum_pyarrow_version()
super().__init__(udf_descs, runner_conf, eval_conf)
# See TransformWithStateInPandasExec for how arg_offsets are used to
# distinguish between grouping attributes and data attributes.
# parsed offsets:
# [
# [groupingKeyOffsets, dedupDataOffsets],
# [initStateGroupingOffsets, dedupInitDataOffsets]
# ]
parsed_offsets = extract_key_value_indexes(self._udf_desc.args_offsets)
self._key_offsets = parsed_offsets[0][0]
self._init_key_offsets = parsed_offsets[1][0]
self._stateful_processor_api_client = StatefulProcessorApiClient(
eval_conf.state_server_socket_port, # type: ignore[arg-type]
eval_conf.grouping_key_schema, # type: ignore[arg-type]
auth_secret=eval_conf.state_server_auth_secret,
)
arrow_max_records_per_batch = runner_conf.arrow_max_records_per_batch
self._arrow_max_records_per_batch = (
arrow_max_records_per_batch if arrow_max_records_per_batch > 0 else 2**31 - 1
)
def run(self, split_index: int, data: Iterator[pa.RecordBatch]) -> Iterator[pa.RecordBatch]:
"""Apply transformWithStateInPySpark UDF with initial state over Rows.
The input batches carry two struct columns, ``inputData`` and
``initState``; each batch holds one or the other but never both.
Rows are flattened out of whichever struct is present, regrouped by
grouping key, and re-chunked into Row lists bounded by
arrow_max_records_per_batch. The UDF is invoked once per chunk with
two separate iterators (data Rows and init-state Rows), then once for
PROCESS_TIMER and once for COMPLETE. The UDF yields (iterator of Row,
Spark type) pairs that are converted back into Arrow RecordBatches
wrapped in a single struct column for the output stream.
"""
import pyarrow as pa
def extract_rows(
cur_batch: pa.RecordBatch, col_name: str, offsets: list
) -> Optional[Iterator[tuple[Any, Any]]]:
data_column = cur_batch.column(cur_batch.schema.get_field_index(col_name))
# Check if the entire column is null.
if data_column.null_count == len(data_column):
return None
data_field_names = [
data_column.type[i].name for i in range(data_column.type.num_fields)
]
data_field_arrays = [data_column.field(i) for i in range(data_column.type.num_fields)]
DataRow = Row(*data_field_names)
table = pa.Table.from_arrays(data_field_arrays, names=data_field_names)
if table.num_rows == 0:
return None
def row_iterator() -> Iterator[tuple[Any, Any]]:
for row_idx in range(table.num_rows):
key = tuple(table.column(o)[row_idx].as_py() for o in offsets)
row = DataRow(
*(table.column(i)[row_idx].as_py() for i in range(table.num_columns))
)
yield (key, row)
return row_iterator()
def row_stream() -> Iterator[tuple[Any, Optional[Any], Optional[Any]]]:
# The arrow batch is written in the schema:
# schema: StructType = new StructType()
# .add("inputData", dataSchema)
# .add("initState", initStateSchema)
# We parse each batch into tuples of (key, inputData, initState).
# Each batch will have either init_data or input_data, not both.
for batch in data:
input_result = extract_rows(batch, "inputData", self._key_offsets)
init_result = extract_rows(batch, "initState", self._init_key_offsets)
assert not (input_result is not None and init_result is not None)
if input_result is not None:
for key, input_data_row in input_result:
yield (key, input_data_row, None)
elif init_result is not None:
for key, init_state_row in init_result:
yield (key, None, init_state_row)
def generate_data_batches() -> Iterator[tuple[Any, tuple[Any, Any]]]:
"""
Deserialize ArrowRecordBatches and return a generator of
(grouping key, (data Rows iterator, init-state Rows iterator))
chunks bounded by arrow_max_records_per_batch.
This function must avoid materializing multiple Arrow
RecordBatches into memory at the same time, and data chunks
from the same grouping key should appear sequentially.
"""
for k, group_rows in itertools.groupby(row_stream(), key=lambda x: x[0]):
input_rows: list = []
init_rows: list = []
for _, input_row, init_row in group_rows:
if input_row is not None:
input_rows.append(input_row)
if init_row is not None:
init_rows.append(init_row)
total_len = len(input_rows) + len(init_rows)
if total_len >= self._arrow_max_records_per_batch:
yield (k, (iter(input_rows), iter(init_rows)))
input_rows = []
init_rows = []
if input_rows or init_rows:
yield (k, (iter(input_rows), iter(init_rows)))
def convert_results(result_rows: Iterable[Any]) -> Iterator[pa.RecordBatch]:
# TODO(SPARK-XXXXX): add verification that elements in result_rows
# are indeed of type Row and conform to assigned cols
# Convert spark type to arrow type
# TODO: we need to make this configurable, currently using default values.
arrow_type = to_arrow_type(
self._udf_desc.return_type,
timezone="UTC",
prefers_large_types=False,
)
rows_as_dict = [row.asDict(True) for row in result_rows]
pdf_schema = pa.schema(list(arrow_type))
record_batch = pa.RecordBatch.from_pylist(rows_as_dict, schema=pdf_schema)
yield ArrowBatchTransformer.wrap_struct(record_batch)
for key, group in itertools.groupby(generate_data_batches(), key=lambda x: x[0]):
# These must be generator expressions - do not materialize.
for _, (values_gen, init_states_gen) in group:
yield from convert_results(
self._func(
self._stateful_processor_api_client,
TransformWithStateInPandasFuncMode.PROCESS_DATA,
key,
values_gen,
init_states_gen,
)
)
yield from convert_results(
self._func(
self._stateful_processor_api_client,
TransformWithStateInPandasFuncMode.PROCESS_TIMER,
None,
iter([]),
iter([]),
)
)
yield from convert_results(
self._func(
self._stateful_processor_api_client,
TransformWithStateInPandasFuncMode.COMPLETE,
None,
iter([]),
iter([]),
)
)