blob: 3a003321783fd15bf3368c08acd138ba9b888e50 [file]
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//! Experimental SDK for native data sources in Rust, over version 1 of the C ABI of Spark,
//! `spark_datasource.h`.
//!
//! Implement [`DataSource`], naming the types of the operations that it implements: a
//! [`ScanBuilder`] for batch reads, a [`MicroBatchStream`] for micro-batch streaming reads, a
//! [`BatchWrite`] for batch writes, and a [`StreamingWrite`] for streaming writes, or
//! [`Unsupported`] for an operation that it does not implement. Then invoke
//! [`export_data_source!`] once in a `cdylib` crate. The SDK
//! owns handles, plan/message serialization, C ABI adaptation, and Arrow import/export.
//! Rust traits stay inside the plugin; they are not an ABI between dynamic libraries.
//!
//! Methods receive their inputs in `#[non_exhaustive]` structs: `*Info` structs on the driver
//! ([`CreateInfo`], [`ScanInfo`], [`MicroBatchStreamInfo`], [`LatestOffsetInfo`],
//! [`MicroBatchInfo`], [`CommitOffsetInfo`], [`WriteInfo`], [`CommitInfo`], [`AbortInfo`],
//! [`StreamingWriteInfo`], [`StreamingCommitInfo`], and [`StreamingAbortInfo`]), `*Context`
//! structs, which carry the live [`TaskContext`], on the executors ([`ReaderContext`] and
//! [`WriterContext`]), and [`HostInfo`], which [`DataSource::init`] receives in every JVM
//! that loads the library, executors included. The methods whose function of the header
//! passes nothing but the size of its info struct yet, [`ScanBuilder::plan`],
//! [`MicroBatchStream::initial_offset`], and the `commit` and `abort` of [`DataWriter`], take
//! no struct: a field added there would be a breaking change of the SDK.
use std::collections::BTreeMap;
use arrow_array::RecordBatch;
use arrow_schema::{ArrowError, SchemaRef};
use serde::de::DeserializeOwned;
use serde::Serialize;
mod abi;
mod bridge;
mod capabilities;
mod export;
mod host;
mod task;
mod unsupported;
mod write;
/// Implementation details of [`export_data_source!`] and of the SDK's own tests, not an API.
#[doc(hidden)]
pub mod __private {
pub use crate::bridge::Bridge;
pub use crate::export::{library, Exported};
pub mod abi {
pub use crate::abi::*;
}
}
/// The Arrow crate of [`RecordBatch`], which connectors can use to have the same version of
/// Arrow as the SDK.
pub use arrow_array;
/// The Arrow crate of [`SchemaRef`] and [`ArrowError`], which connectors can use to have the
/// same version of Arrow as the SDK.
pub use arrow_schema;
pub use capabilities::Capabilities;
pub use host::HostInfo;
/// The `log` crate, whose macros write to the log of the Spark driver or executor, with
/// the path of the library, once Spark initializes the library.
///
/// Any thread can call them, including in the dependencies of the library. Outside
/// Spark, for example in unit tests, their messages are dropped. The SDK sets the level of
/// the `log` crate to that of the log of Spark, such as `Debug` when Spark logs debug
/// messages, so the library does not set it.
pub use log;
pub use task::TaskContext;
pub use unsupported::Unsupported;
pub type Error = Box<dyn std::error::Error + Send + Sync>;
pub type Result<T> = std::result::Result<T, Error>;
pub type Options = BTreeMap<String, String>;
// Data sources: Spark creates a DataSource for each operation, and its associated types
// name the operations that the source implements.
/// What Spark passes to create a data source.
///
/// Use [`Self::new`] to construct this value outside Spark, and include `..` when
/// destructuring it to allow additional fields.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct CreateInfo {
/// The name of the data source, as listed in the manifest of its package, or in
/// lower case for an installed library. A library that implements several data
/// sources, installed or listed under each of their names, can choose by this name.
pub name: String,
/// The options, with keys in lower case. A capability probe uses the options of the
/// table; an operation may override them with the options of its query.
pub options: Options,
}
impl CreateInfo {
/// The information of a source created outside Spark, for example in a test.
pub fn new(name: impl Into<String>, options: Options) -> Self {
Self {
name: name.into(),
options,
}
}
}
/// Driver-side source configuration, schema discovery, and the operations of a source.
///
/// The associated types name the operations that the source implements, and
/// [`Unsupported`] stands for one that it does not implement:
///
/// ```rust,ignore
/// impl DataSource for RangeSource {
/// type ScanBuilder = RangeScan;
/// type MicroBatchStream = RangeStream;
/// type BatchWrite = Unsupported;
/// type StreamingWrite = Unsupported;
/// ...
/// }
/// ```
///
/// The SDK installs native callbacks and reports capabilities only for the implemented
/// operations, and calls the `new_*` method of an operation only for them.
///
/// Spark creates a source for each schema inference, scan, stream, and write, including
/// the write of each micro-batch, and may create one only to inspect its capabilities,
/// then drop it. Keep construction lightweight. Scan builders, streams, and writes cannot
/// borrow their source, which Spark drops first.
pub trait DataSource: Send + Sized + 'static {
fn new(info: CreateInfo) -> Result<Self>;
/// Called once in each JVM when Spark loads the library, before it creates any
/// source. On failure, Spark does not use the library, and the error is logged.
fn init(_host: &HostInfo) -> Result<()> {
Ok(())
}
// Schema discovery.
/// Whether [`infer_schema`](Self::infer_schema) infers the schema of the data. When
/// false, the SDK leaves the native `schema` function unset: Spark then requires a
/// user-specified schema, reports its standard error without one, and never calls
/// `infer_schema`.
const INFERS_SCHEMA: bool = true;
/// Returns the schema of the data, when the user does not specify one.
fn infer_schema(&self) -> Result<SchemaRef> {
Err(
"DataSource::infer_schema is not implemented: implement it, or set \
DataSource::INFERS_SCHEMA to false to require a user-specified schema"
.into(),
)
}
// Batch reads.
/// The scan builder of batch reads, which [`new_scan`](Self::new_scan) creates, or
/// [`Unsupported`].
type ScanBuilder: ScanBuilder;
/// Creates an independent scan of [`ScanInfo::schema`], which the user may have
/// specified; reject unsupported schemas.
fn new_scan(&self, _info: ScanInfo) -> Result<Self::ScanBuilder> {
Err(
"DataSource::new_scan is not implemented: implement it, or set \
DataSource::ScanBuilder to Unsupported"
.into(),
)
}
// Micro-batch streaming reads.
/// The stream of micro-batch streaming reads, which
/// [`new_micro_batch_stream`](Self::new_micro_batch_stream) creates, or [`Unsupported`].
type MicroBatchStream: MicroBatchStream;
/// Creates the stream of a streaming query that reads the columns of
/// [`MicroBatchStreamInfo::schema`]; reject unsupported schemas.
fn new_micro_batch_stream(
&self,
_info: MicroBatchStreamInfo,
) -> Result<Self::MicroBatchStream> {
Err(
"DataSource::new_micro_batch_stream is not implemented: implement it, or set \
DataSource::MicroBatchStream to Unsupported"
.into(),
)
}
// Batch writes.
/// The coordinator of batch writes, which [`new_write`](Self::new_write) creates, or
/// [`Unsupported`].
type BatchWrite: BatchWrite;
/// Validate the input schema and create an independent write. Construction may be
/// followed only by Drop, without commit or abort, so it must not make irreversible
/// changes. The SDK calls this only while the instance's capabilities include
/// `BATCH_WRITE`, and passes [`WriteMode::Truncate`] only when they include
/// `BATCH_TRUNCATE`, which the write declares with [`BatchWrite::SUPPORTS_TRUNCATE`].
fn new_write(&self, _info: WriteInfo) -> Result<Self::BatchWrite> {
Err(
"DataSource::new_write is not implemented: implement it, or set \
DataSource::BatchWrite to Unsupported"
.into(),
)
}
// Streaming writes.
/// The coordinator of the write of each micro-batch of a streaming query, which
/// [`new_streaming_write`](Self::new_streaming_write) creates, or [`Unsupported`].
type StreamingWrite: StreamingWrite;
/// Validate the input schema and create the write of a micro-batch. As with
/// [`new_write`](Self::new_write), construction may be followed only by Drop, so it must
/// not make irreversible changes. The SDK calls this only while the instance's
/// capabilities include `STREAMING_WRITE`, and passes [`WriteMode::Truncate`] only when
/// they include `STREAMING_TRUNCATE`, which the write declares with
/// [`StreamingWrite::SUPPORTS_TRUNCATE`].
fn new_streaming_write(&self, _info: StreamingWriteInfo) -> Result<Self::StreamingWrite> {
Err(
"DataSource::new_streaming_write is not implemented: implement it, or set \
DataSource::StreamingWrite to Unsupported"
.into(),
)
}
// Instance restrictions.
/// Returns the operations of this instance. The SDK keeps only those that the source
/// implements, so the default, [`Capabilities::all`], keeps them all. Override this to
/// restrict an instance, for example for a read-only destination. The SDK evaluates
/// it once when Spark creates the source. An instance without operations still
/// supports schema discovery.
fn capabilities(&self) -> Capabilities {
Capabilities::all()
}
}
// Batch reads: a ScanBuilder negotiates pushdown and plans the scan on the driver, and a
// PartitionReaderFactory opens a reader of each partition on the executors.
/// What Spark passes to create a scan.
///
/// Use [`Self::new`] to construct this value outside Spark, and include `..` when
/// destructuring it to allow additional fields.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct ScanInfo {
/// The schema to read: the schema of the source, or one that the user specified.
pub schema: SchemaRef,
}
impl ScanInfo {
/// The information of a scan created outside Spark, for example in a test.
pub fn new(schema: SchemaRef) -> Self {
Self { schema }
}
}
/// Negotiates pushdown on the driver, then plans the scan.
///
/// Spark calls [`push_limit`](Self::push_limit) and [`prune_columns`](Self::prune_columns),
/// each at most once and in this order, then [`plan`](Self::plan), and drops the builder.
/// After a call fails, it makes no other call. It drops a builder without planning it when it
/// does not read the scan, for example to only explain a query, once the scan is no longer
/// referenced, which may be late and on another thread, so a builder must not hold a resource
/// that other operations need exclusively, such as a lock, until Drop. Only the serialized
/// factory and partitions reach the executors.
pub trait ScanBuilder: Send + Sized + 'static {
type Factory: PartitionReaderFactory;
/// False only for [`Unsupported`], from which the SDK derives the capabilities.
#[doc(hidden)]
const SUPPORTED: bool = true;
/// Pushes a LIMIT of `limit` rows down to the scan. Return [`LimitSupport::Global`] only
/// if the whole scan then returns at most `limit` rows. Spark pushes a limit only when the
/// query does not filter the rows, since it filters them after reading.
fn push_limit(&mut self, _limit: usize) -> Result<LimitSupport> {
Ok(LimitSupport::Unsupported)
}
/// Tells the scan the top-level columns that the query needs, in order, so that it can
/// plan to read only them. There may be none, for example for `SELECT count(*)`: the
/// batches then only have a number of rows. Readers get these columns in
/// [`ReaderContext::schema`] whether or not the scan implements this method.
fn prune_columns(&mut self, _columns: &[String]) -> Result<()> {
Ok(())
}
fn plan(self) -> Result<ScanPlan<Self::Factory>>;
}
/// Whether a connector limits the rows produced by the whole scan.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum LimitSupport {
/// The scan may return more rows than the limit. A connector may still cap the
/// rows of each partition on its own: Spark applies its limit after reading.
Unsupported,
/// The whole scan returns at most the limit. Spark still applies its limit.
Global,
}
/// A driver-created plan containing only serializable values.
///
/// The SDK serializes the reader factory once and each partition independently.
/// Include snapshot identifiers when needed for repeatable task attempts. Do not
/// put pointers, open files, connections, or process-local resources here.
#[non_exhaustive]
pub struct ScanPlan<F: PartitionReaderFactory> {
pub factory: F,
pub partitions: Vec<F::Partition>,
}
impl<F: PartitionReaderFactory> ScanPlan<F> {
pub fn new(factory: F, partitions: Vec<F::Partition>) -> Self {
Self {
factory,
partitions,
}
}
}
/// Serializable configuration from which executors open partition readers.
///
/// The factory carries configuration or a snapshot descriptor; open connections and
/// other live resources in [`open`](Self::open). Spark passes the schema to read with
/// each partition, so the factory need not carry it.
pub trait PartitionReaderFactory: Serialize + DeserializeOwned + Send + 'static {
type Partition: Serialize + DeserializeOwned + Send + 'static;
/// Opens a lazy reader of the batches of `partition`, with the columns of
/// [`ReaderContext::schema`].
///
/// - Return batches lazily, with bounded memory.
/// - The reader owns its resources and cannot borrow this short-lived factory. It
/// may keep `context` until Spark releases the stream.
/// - Spark may call the reader on different threads, never concurrently, so it must
/// be `Send` but not necessarily `Sync`.
/// - Check [`TaskContext::is_interrupted`] during long operations, and use timeouts
/// for blocking I/O.
/// - Batches must own their buffers, because Spark may release the stream before
/// its batches. Releasing a batch drops its references to its buffers, which runs the
/// Drop of the owner of a buffer, such as one that returns memory to a pool, when they
/// are the last ones. Spark may release a batch after the task ended, on another thread,
/// such as a JVM Cleaner thread after a LIMIT, so that Drop should be quick and not
/// block. The README's Scope section describes panics there.
/// - Spark may open the same partition at the same time in several attempts of its
/// task, so a reader must not take a resource that another attempt needs exclusively.
///
/// The SDK checks Spark-compatible Arrow types, the schema to read and its
/// nullability, the precision of decimal values, and zero array offsets recursively. It
/// reports incompatible layouts instead of copying buffers. After a batch fails these
/// checks, or the reader returns an error, the stream fails every later call with the
/// same error. A batch may have more columns than the schema to read,
/// such as all the columns of the source: the SDK keeps those of the schema, by name.
/// Batches may use any supported Arrow encoding of a Spark type, such as `LargeUtf8`
/// for a string, but the first batch determines the Arrow schema of the stream, which
/// every later batch must have.
fn open<'task>(
&self,
partition: Self::Partition,
context: ReaderContext<'task>,
) -> Result<PartitionReader<'task>>;
}
/// What an executor reader receives with its partition.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct ReaderContext<'task> {
/// The schema to read: the columns that the scan reads, in order, which are those of
/// [`ScanBuilder::prune_columns`] for a batch scan, and of
/// [`MicroBatchStreamInfo::schema`] for a micro-batch.
pub schema: SchemaRef,
/// The index of this partition in the scan or micro-batch's [`ScanPlan::partitions`].
/// This is not the Spark task's partition ID: one task may read several partitions,
/// for example after a coalesce or union. The index is unique only within that plan.
pub partition_index: i32,
/// The task that reads the partition.
pub task: TaskContext<'task>,
}
impl<'task> ReaderContext<'task> {
/// The context of a reader opened outside Spark, for example in a test.
pub fn new(schema: SchemaRef, partition_index: i32, task: TaskContext<'task>) -> Self {
Self {
schema,
partition_index,
task,
}
}
}
/// The lazy batches of one partition, which [`PartitionReaderFactory::open`] returns.
/// Any Arrow `RecordBatchReader`, such as a Parquet reader, is such an iterator.
pub type PartitionReader<'task> =
Box<dyn Iterator<Item = std::result::Result<RecordBatch, ArrowError>> + Send + 'task>;
// Micro-batch streaming reads: a MicroBatchStream finds new data and plans each micro-batch on
// the driver, and the PartitionReaderFactory of each plan reads its partitions on the executors,
// as in batch reads.
/// What Spark passes to create the stream of a streaming query.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct MicroBatchStreamInfo {
/// The columns to read: those of the schema of the source, or of one that the user
/// specified, or only the top-level columns of it that the query needs. The readers of
/// the micro-batches get it in [`ReaderContext::schema`].
pub schema: SchemaRef,
/// A directory in the checkpoint of the query that is reserved for this source, such as
/// `file:/tmp/checkpoint/sources/0`, where the stream may keep what it needs to recover
/// from a failure. It stays the same when the query restarts. It is a fully qualified
/// Hadoop path, whose characters are not escaped as in a URI, on a file system that the
/// library may not be able to access, such as `s3a`. Spark keeps what it needs there under
/// the names that start with `_spark`, which the library must not use.
pub checkpoint_location: String,
}
impl MicroBatchStreamInfo {
/// The information of a stream created outside Spark, for example in a test.
pub fn new(schema: SchemaRef, checkpoint_location: impl Into<String>) -> Self {
Self {
schema,
checkpoint_location: checkpoint_location.into(),
}
}
}
/// Finds new data and plans the micro-batches of a streaming query, on the driver.
///
/// Spark creates a stream when a streaming query starts, or restarts from its checkpoint,
/// and drops it when the query stops. For each micro-batch, it asks for the
/// [`latest_offset`](Self::latest_offset), and when that differs from the end of the
/// previous micro-batch, [`plan`](Self::plan)s the data in between. Only the serialized
/// factory and partitions of each plan reach the executors, which read them as those of a
/// batch scan.
///
/// # Offsets
///
/// An offset is a progress boundary whose representation and mapping to records the connector
/// defines. A micro-batch reads the data between its starting and ending boundaries. For example,
/// when an offset stores the next row to read, `{"next":10}` to `{"next":13}` reads rows 10, 11,
/// and 12: the row range is `[10, 13)`. Equal boundaries describe no data, and adjacent ranges
/// must neither repeat nor skip records.
///
/// Spark stores offsets in the checkpoint of the query, as the JSON of
/// [`Offset`](Self::Offset), and compares them by that JSON, so serialize a position to the
/// same JSON each time: for example, use structs and `BTreeMap` rather than `HashMap`. The
/// SDK stores exactly that JSON, and later versions will not wrap or re-encode it: the format
/// of an offset is that of the connector's type, which outlives any build of the library. A
/// restarted query passes the offsets of its checkpoint, which an earlier version of the
/// library may have written, to `latest_offset`, `plan`, and `commit`. When that version had a
/// different `Offset` type, the same position may encode to different JSON, so Spark runs one
/// micro-batch without rows.
///
/// JSON has no NaN or infinite numbers: serde writes such a float as `null`. The SDK decodes the
/// JSON of each offset that the stream returns before Spark uses or stores it, so the call fails
/// when a NaN or an infinity is in an `f32` or `f64` field, which does not decode from `null`,
/// but an `Option<f64>` field that holds one decodes as `None`.
///
/// # Admission control
///
/// [`default_read_limit`](Self::default_read_limit) limits the data of each micro-batch,
/// typically with an option of the source. Spark passes the limit to `latest_offset`, except
/// when a micro-batch reads all the available data, as with `Trigger.Once`. With
/// `Trigger.AvailableNow`, a run reads the data that was available when it started: with a
/// limit, in several micro-batches, whose latest offsets must not go beyond
/// [`LatestOffsetInfo::run_end`].
///
/// # Lifecycle
///
/// - Calls never run concurrently.
/// - After a call fails, including with a panic, Spark makes no other call: it stops the
/// query, and drops the stream.
/// - After a restart, Spark may plan the last range of the query again, so `plan` must
/// return partitions that read the same data for the same range.
pub trait MicroBatchStream: Send + 'static {
/// A progress boundary in the data, such as the next record of each partition of a log.
type Offset: Serialize + DeserializeOwned;
/// The reader factory of the micro-batches, which can be that of batch scans.
type Factory: PartitionReaderFactory;
/// False only for [`Unsupported`], from which the SDK derives the capabilities.
#[doc(hidden)]
const SUPPORTED: bool = true;
/// Returns the starting progress boundary when the query has no checkpoint. Spark keeps
/// it in the checkpoint, and asks only when the checkpoint does not have it yet, so a query
/// that restarts before its first micro-batch is committed starts from the same offset.
fn initial_offset(&mut self) -> Result<Self::Offset>;
/// Returns the limit on the data of each micro-batch. Spark asks once, when the query
/// starts. The default, [`ReadLimit::AllAvailable`], reads all the available data in
/// each micro-batch.
fn default_read_limit(&self) -> Result<ReadLimit> {
Ok(ReadLimit::AllAvailable)
}
/// Returns the end of the next micro-batch: the offset of the most recent data after
/// [`LatestOffsetInfo::start`], within its limit, and not beyond its end of the run.
/// Return `start` when there is no new data. When there is, return an offset after
/// `start`, even if the next unit of data, such as a file, is larger than the limit:
/// Spark sees no new data otherwise, and never reads further.
///
/// With a limit, Spark also asks without a limit for some micro-batches, to report the
/// progress of the query, so do not count the calls as micro-batches, or reserve data in
/// them.
fn latest_offset(&mut self, info: LatestOffsetInfo<Self::Offset>) -> Result<Self::Offset>;
/// Plans the micro-batch between the progress boundaries [`MicroBatchInfo::start`] and
/// [`MicroBatchInfo::end`], using the connector's mapping from offsets to records.
fn plan(&mut self, info: MicroBatchInfo<Self::Offset>) -> Result<ScanPlan<Self::Factory>>;
/// Tells the stream that Spark has processed all the data before the progress boundary
/// [`CommitOffsetInfo::end`] and will not read that data again, so the stream can release it.
fn commit(&mut self, _info: CommitOffsetInfo<Self::Offset>) -> Result<()> {
Ok(())
}
}
/// A limit on the data of a micro-batch.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
#[non_exhaustive]
pub enum ReadLimit {
/// All the available data.
AllAvailable,
/// A maximum number of rows, from 1 through [`i64::MAX`]. The SDK rejects values outside
/// this range at the native ABI boundary. A stream may exceed this limit when its next
/// unit of data is larger, as described by [`MicroBatchStream::latest_offset`].
MaxRows(u64),
}
/// What Spark passes to ask a stream for its latest offset.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct LatestOffsetInfo<O> {
/// The starting progress boundary: the end of the previous micro-batch, or the initial
/// offset.
pub start: O,
/// The limit on the data after `start`: that of
/// [`MicroBatchStream::default_read_limit`], or [`ReadLimit::AllAvailable`] when Spark
/// reads all the available data, such as for the progress of the query or with
/// `Trigger.Once`.
pub limit: ReadLimit,
/// With `Trigger.AvailableNow` and a limit, the end of the run: the latest offset must not
/// go beyond it, so that the run reads the data that was available when it started. It is
/// not the end of the next micro-batch. `None` otherwise.
pub run_end: Option<O>,
}
impl<O> LatestOffsetInfo<O> {
/// A request without a limit, made outside Spark, for example in a test. Set
/// [`limit`](Self::limit) and [`run_end`](Self::run_end) for other requests.
pub fn new(start: O) -> Self {
Self {
start,
limit: ReadLimit::AllAvailable,
run_end: None,
}
}
}
/// What Spark passes to plan a micro-batch: the offsets that delimit its data.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct MicroBatchInfo<O> {
/// The starting progress boundary: the end of the previous micro-batch, or the initial
/// offset. See [`MicroBatchStream`]'s offset contract for how boundaries map to records.
pub start: O,
/// The ending progress boundary, which [`MicroBatchStream::latest_offset`] returned.
pub end: O,
}
impl<O> MicroBatchInfo<O> {
/// The information of a micro-batch planned outside Spark, for example in a test.
pub fn new(start: O, end: O) -> Self {
Self { start, end }
}
}
/// What Spark passes to commit the data of a stream.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct CommitOffsetInfo<O> {
/// The progress boundary of the processed data, which Spark will not read again.
pub end: O,
}
impl<O> CommitOffsetInfo<O> {
/// The information of a commit made outside Spark, for example in a test.
pub fn new(end: O) -> Self {
Self { end }
}
}
// Batch writes: a BatchWrite coordinates the write on the driver, and a DataWriterFactory
// opens a DataWriter for each task attempt on the executors.
/// What Spark passes to create a batch write.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct WriteInfo {
/// The schema of the data to write.
pub schema: SchemaRef,
/// How the write changes the destination.
pub mode: WriteMode,
}
impl WriteInfo {
/// The information of a write created outside Spark, for example in a test.
pub fn new(schema: SchemaRef, mode: WriteMode) -> Self {
Self { schema, mode }
}
}
/// How a write, or the write of a micro-batch, changes the destination.
///
/// New modes may be added. The SDK passes a mode only when the source instance
/// has the matching capability, so a source never receives a mode it did not
/// opt in to; a wildcard arm can return an error.
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum WriteMode {
Append,
/// Replace existing data at driver commit, never at writer creation.
Truncate,
}
/// A driver coordinator for one batch write, which also creates the serializable
/// configuration of its task writers.
///
/// Keep irreversible publication, including removing overwritten data, in commit.
/// Drop releases resources; it must not publish data or perform fallible flushing.
///
/// # Lifecycle
///
/// Spark guarantees the following callback order:
///
/// - `new_writer_factory` runs once, right after [`DataSource::new_write`], before any task
/// opens a writer. When it fails, the SDK cannot serialize its factory, or Spark cannot
/// copy the serialized factory, for example when it is too large, the write fails and
/// the coordinator is dropped without `abort`, so Drop must also release what only this
/// write staged.
/// - `commit` runs at most once, when every task succeeded. The SDK fails the write
/// without calling it when a task message cannot be decoded.
/// - `abort` runs at most once, never after a successful `commit`. It follows a failed
/// `commit`, or a failed task, also when computing the data to write failed.
/// - Drop runs exactly once, otherwise when Spark closes the writer: right after a
/// successful `commit` or after `abort`, or, for a write that Spark neither commits nor
/// aborts, possibly late and on another thread.
/// - A panic in any method is reported as that method's failure.
///
/// These guarantees do not determine the outcome of a remote transaction. A destination
/// may publish the write even when `commit` returns an error, for example if its
/// acknowledgement is lost. Use a durable operation identity to check the destination's
/// outcome and make commit and cleanup idempotent. `abort` and Drop must preserve published
/// output, including output whose commit was not acknowledged. If the outcome cannot be
/// resolved, leave potentially published data intact for later recovery.
pub trait BatchWrite: Send + 'static {
/// The configuration from which each task attempt opens its writer.
type Factory: DataWriterFactory<Message = Self::Message>;
/// The message with which a task attempt reports its staged output to the driver.
type Message: Serialize + DeserializeOwned + Send + 'static;
/// Whether the write supports [`WriteMode::Truncate`], for the save mode "overwrite",
/// which adds the `BATCH_TRUNCATE` capability.
const SUPPORTS_TRUNCATE: bool = false;
/// False only for [`Unsupported`], from which the SDK derives the capabilities.
#[doc(hidden)]
const SUPPORTED: bool = true;
/// Returns the configuration of the task writers, which the SDK serializes for them.
/// Include a unique operation ID when needed; Spark supplies no batch query ID.
fn new_writer_factory(&self) -> Result<Self::Factory>;
/// Publish only the winning attempt from each input partition, whose messages
/// [`CommitInfo::messages`] has. Task retries and speculative attempts can stage other
/// data that must not be published.
fn commit(&mut self, info: CommitInfo<Self::Message>) -> Result<()>;
/// Abort after input, task, or driver commit failure, with the messages that reached the
/// driver in [`AbortInfo::messages`]. Executor loss can leave orphaned staged data; plan
/// for its cleanup.
fn abort(&mut self, info: AbortInfo<Self::Message>) -> Result<()>;
}
/// What Spark passes to commit a batch write.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct CommitInfo<M> {
/// The message of the winning task attempt of each input partition, at the position of
/// its partition.
pub messages: Vec<M>,
}
impl<M> CommitInfo<M> {
/// The information of a commit made outside Spark, for example in a test.
pub fn new(messages: Vec<M>) -> Self {
Self { messages }
}
}
/// What Spark passes to abort a batch write.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct AbortInfo<M> {
/// The message of each input partition, at the position of its partition, or `None` for a
/// partition whose commit message did not reach the driver, or cannot be decoded.
pub messages: Vec<Option<M>>,
}
impl<M> AbortInfo<M> {
/// The information of an abort made outside Spark, for example in a test.
pub fn new(messages: Vec<Option<M>>) -> Self {
Self { messages }
}
}
/// Serializable configuration for independent task-attempt writers, which
/// [`BatchWrite::new_writer_factory`] or [`StreamingWrite::new_writer_factory`] creates.
///
/// Open connections and files in open, not in the factory. A returned writer
/// cannot borrow this temporary factory, but may retain the context until
/// close. Callbacks can move between threads sequentially; writers must be Send.
pub trait DataWriterFactory: Serialize + DeserializeOwned + Send + 'static {
type Message: Serialize + DeserializeOwned + Send + 'static;
fn open<'task>(
&self,
context: WriterContext<'task>,
) -> Result<PartitionWriter<'task, Self::Message>>;
}
/// What a task writer receives when it opens.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct WriterContext<'task> {
/// The schema of the batches to write, as in [`WriteInfo::schema`].
pub schema: SchemaRef,
/// The Spark task's partition ID. Its winning commit message has this position in
/// [`CommitInfo::messages`] or [`StreamingCommitInfo::messages`].
pub partition_id: i32,
/// The task attempt that writes.
pub task: TaskContext<'task>,
/// The epoch of the micro-batch of a streaming write, or `None` for a batch write.
pub epoch_id: Option<i64>,
}
impl<'task> WriterContext<'task> {
/// The context of a batch writer opened outside Spark, for example in a test. Set
/// [`epoch_id`](Self::epoch_id) for a streaming writer.
pub fn new(schema: SchemaRef, partition_id: i32, task: TaskContext<'task>) -> Self {
Self {
schema,
partition_id,
task,
epoch_id: None,
}
}
}
/// One task attempt's writer. Spark drops it after commit or abort.
///
/// # Lifecycle
///
/// Spark guarantees the following callback order:
///
/// - `write` runs zero or more times, before `commit` or `abort`, and never after a
/// call has failed.
/// - `commit` runs at most once, never after a failed `write` or after `abort`.
/// - `abort` runs at most once, never after a `commit` whose message reached Spark.
/// It may follow a failed `write` or `commit`, including a `commit` that returned
/// a message that the SDK could not serialize, so it must handle a task attempt
/// that already finished its output.
/// - Drop runs exactly once when Spark closes the writer, possibly without
/// `commit` or `abort`.
/// - The writer of a task attempt that Spark kills, such as a speculative one, may still
/// write, abort, and drop after the driver committed or aborted the write.
/// - A panic in any method is reported as that method's failure.
pub trait DataWriter: Send {
type Message;
/// Takes ownership of an imported Arrow batch, which may be retained after this call,
/// but not after the writer is dropped: its buffers belong to the data writer of Spark,
/// and the ABI requires the library to release them before Spark closes the writer, so do
/// not pass the batch to a thread or a value that outlives the writer. Spark cannot free a
/// batch that is still retained when it closes the writer: it logs an error, and the memory
/// stays allocated until the batch is dropped. It has the schema of
/// [`WriterContext::schema`], and at most `spark.sql.execution.arrow.maxRecordsPerBatch`
/// rows. When Spark writes the batches of a native scan without converting them to rows,
/// it copies their values as the scan returned them, without the checks of a conversion,
/// such as the precision of a decimal. The SDK validates the layout and values of each batch
/// before this call, and fails the write of one that Arrow rejects, such as a Spark string
/// that is not UTF-8. It does not check nullability, which Spark checks masked by the
/// parent, so a field that is not nullable can still hold nulls under a null parent.
/// Process batches incrementally and check task cancellation.
fn write(&mut self, batch: RecordBatch) -> Result<()>;
/// Finish staging and return a typed message for the driver. Do not make the
/// whole write visible here. On failure (including message serialization),
/// the SDK keeps this writer available for abort.
fn commit(&mut self) -> Result<Self::Message>;
fn abort(&mut self) -> Result<()>;
}
/// The writer of one task attempt, which [`DataWriterFactory::open`] returns.
pub type PartitionWriter<'task, M> = Box<dyn DataWriter<Message = M> + 'task>;
// Streaming writes: a StreamingWrite coordinates the write of one micro-batch on the driver,
// and its DataWriterFactory opens a DataWriter for each task attempt on the executors, as in
// batch writes.
/// What Spark passes to create the write of a micro-batch.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct StreamingWriteInfo {
/// The schema of the data to write.
pub schema: SchemaRef,
/// How the micro-batch changes the destination: [`WriteMode::Truncate`] for the output
/// mode "complete", which replaces the output of the previous micro-batches.
pub mode: WriteMode,
/// The ID of the streaming query, which stays the same when the query restarts from
/// its checkpoint.
pub query_id: String,
}
impl StreamingWriteInfo {
/// The information of a write created outside Spark, for example in a test.
pub fn new(schema: SchemaRef, mode: WriteMode, query_id: impl Into<String>) -> Self {
Self {
schema,
mode,
query_id: query_id.into(),
}
}
}
/// A driver coordinator for the write of one micro-batch, or epoch, of a streaming query,
/// which also creates the serializable configuration of its task writers.
///
/// Spark creates a write for each micro-batch, with the lifecycle of a [`BatchWrite`], and
/// passes the epoch ID of the micro-batch to `commit` and `abort`, in [`StreamingCommitInfo`]
/// and [`StreamingAbortInfo`], and to the task writers in [`WriterContext::epoch_id`]. Spark
/// closes a write that it committed or aborted before it creates the write of the next
/// micro-batch, so the write is dropped by then.
///
/// # Exactly-once output
///
/// Spark may commit an epoch again, with a new write, for example when the query restarts
/// after a failure: committing an epoch that is already committed must have no effect. A
/// restarted query may also abort an epoch that an earlier run committed, so `abort` must
/// only remove what the task writers of its own write wrote. The ID of the query,
/// [`StreamingWriteInfo::query_id`], and the epoch ID identify the output of a micro-batch,
/// also when other queries write to the same destination.
///
/// The remote transaction contract of [`BatchWrite`] applies here too: `abort` and Drop
/// must preserve an epoch published by the current write if its commit acknowledgement
/// was lost. Record the query and epoch identity atomically with publication so a restarted
/// query can determine whether that epoch is already committed.
pub trait StreamingWrite: Send + 'static {
/// The configuration from which each task attempt opens its writer.
type Factory: DataWriterFactory<Message = Self::Message>;
/// The message with which a task attempt reports its staged output to the driver.
type Message: Serialize + DeserializeOwned + Send + 'static;
/// Whether the write supports [`WriteMode::Truncate`], for the output mode "complete",
/// which adds the `STREAMING_TRUNCATE` capability.
const SUPPORTS_TRUNCATE: bool = false;
/// False only for [`Unsupported`], from which the SDK derives the capabilities.
#[doc(hidden)]
const SUPPORTED: bool = true;
/// Returns the configuration of the task writers, which the SDK serializes for them.
fn new_writer_factory(&self) -> Result<Self::Factory>;
/// Publish the winning attempt of each input partition as the output of the epoch
/// [`StreamingCommitInfo::epoch_id`], unless that epoch is already committed.
fn commit(&mut self, info: StreamingCommitInfo<Self::Message>) -> Result<()>;
/// Abort the micro-batch after input, task, or driver commit failure, with the messages
/// that reached the driver in [`StreamingAbortInfo::messages`].
fn abort(&mut self, info: StreamingAbortInfo<Self::Message>) -> Result<()>;
}
/// What Spark passes to commit the write of a micro-batch.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct StreamingCommitInfo<M> {
/// The epoch of the micro-batch, which with [`StreamingWriteInfo::query_id`] identifies
/// its output.
pub epoch_id: i64,
/// The message of the winning task attempt of each input partition, at the position of
/// its partition.
pub messages: Vec<M>,
}
impl<M> StreamingCommitInfo<M> {
/// The information of a commit made outside Spark, for example in a test.
pub fn new(epoch_id: i64, messages: Vec<M>) -> Self {
Self { epoch_id, messages }
}
}
/// What Spark passes to abort the write of a micro-batch.
#[derive(Debug, Clone)]
#[non_exhaustive]
pub struct StreamingAbortInfo<M> {
/// The epoch of the micro-batch.
pub epoch_id: i64,
/// The message of each input partition, at the position of its partition, or `None` for a
/// partition whose commit message did not reach the driver, or cannot be decoded.
pub messages: Vec<Option<M>>,
}
impl<M> StreamingAbortInfo<M> {
/// The information of an abort made outside Spark, for example in a test.
pub fn new(epoch_id: i64, messages: Vec<Option<M>>) -> Self {
Self { epoch_id, messages }
}
}