blob: dee30a91fad9d146f11c31f5677183c5a8479cd1 [file]
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::ffi::{c_char, c_int, c_void, CStr, CString};
use std::mem::size_of;
use std::ptr;
use std::sync::atomic::{AtomicU64, Ordering};
use std::sync::{Arc, Mutex, PoisonError};
use arrow_array::ffi::FFI_ArrowArray;
use arrow_array::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
use arrow_array::{Array, Int64Array, RecordBatch, StructArray};
use arrow_schema::{DataType, Field, Schema, SchemaRef};
use serde::{Deserialize, Serialize};
use spark_datasource::__private::abi::{
self, ArrowSchema, SparkDsCommitOffsetInfo, SparkDsCreateInfo, SparkDsDataWriterInfo,
SparkDsFinishDataWriterInfo, SparkDsFinishWriteInfo, SparkDsHost, SparkDsInitialOffsetInfo,
SparkDsLatestOffsetInfo, SparkDsLibrary, SparkDsPartitionsInfo, SparkDsReadInfo,
SparkDsReadLimit, SparkDsReaderInfo, SparkDsStreamPartitionsInfo, SparkDsStreamReaderInfo,
SparkDsTaskContext, SparkDsWriteInfo, ABI_VERSION,
};
use spark_datasource::{
Capabilities, CommitOffsetInfo, CreateInfo, DataSource, DataWriter, DataWriterFactory,
LatestOffsetInfo, MicroBatchInfo, MicroBatchStream, MicroBatchStreamInfo, PartitionReader,
PartitionReaderFactory, PartitionWriter, ReadLimit, ReaderContext, Result, ScanBuilder,
ScanInfo, ScanPlan, StreamingAbortInfo, StreamingCommitInfo, StreamingWrite,
StreamingWriteInfo, Unsupported, WriterContext,
};
// Each test exercises the actual exported table; this lock isolates fixture observations.
static TEST_LOCK: Mutex<()> = Mutex::new(());
static EVENTS: Mutex<Vec<Event>> = Mutex::new(Vec::new());
// The position of the latest data of the streams.
static LATEST: AtomicU64 = AtomicU64::new(0);
#[derive(Clone, Debug, PartialEq)]
enum Event {
StreamCreated(String),
LatestOffset(u64, ReadLimit, Option<u64>),
OffsetCommitted(u64),
WriteCreated { query_id: String, truncate: bool },
EpochCommitted(i64, Vec<Message>),
EpochAborted(i64, Vec<Option<Message>>),
WriteDropped,
}
fn record(event: Event) {
EVENTS.lock().unwrap().push(event);
}
fn events() -> Vec<Event> {
EVENTS.lock().unwrap().clone()
}
fn schema() -> SchemaRef {
Arc::new(Schema::new(vec![Field::new(
"value",
DataType::Int64,
false,
)]))
}
fn batch(values: Vec<i64>) -> RecordBatch {
RecordBatch::try_new(schema(), vec![Arc::new(Int64Array::from(values))]).unwrap()
}
/// Batch and micro-batch reads, whose reader factories have different types, and streaming
/// writes without batch writes.
struct Source {
failure: String,
capabilities: String,
max_rows: Option<u64>,
}
impl DataSource for Source {
type ScanBuilder = Scan;
type MicroBatchStream = Stream;
type BatchWrite = Unsupported;
type StreamingWrite = Write;
fn new(info: CreateInfo) -> Result<Self> {
let option = |name: &str| info.options.get(name).cloned().unwrap_or_default();
Ok(Self {
failure: option("failure"),
capabilities: option("capabilities"),
max_rows: info
.options
.get("maxrows")
.map(|rows| rows.parse())
.transpose()?,
})
}
fn infer_schema(&self) -> Result<SchemaRef> {
Ok(schema())
}
fn capabilities(&self) -> Capabilities {
match self.capabilities.as_str() {
"no_streaming_write" => Capabilities::all().without(Capabilities::STREAMING_WRITE),
"append_only" => Capabilities::all().without(Capabilities::STREAMING_TRUNCATE),
_ => Capabilities::all(),
}
}
fn new_scan(&self, _: ScanInfo) -> Result<Scan> {
Ok(Scan)
}
fn new_micro_batch_stream(&self, info: MicroBatchStreamInfo) -> Result<Stream> {
if info.schema != schema() {
return Err("unsupported stream schema".into());
}
record(Event::StreamCreated(info.checkpoint_location));
Ok(Stream {
failure: self.failure.clone(),
max_rows: self.max_rows,
})
}
fn new_streaming_write(&self, info: StreamingWriteInfo) -> Result<Write> {
record(Event::WriteCreated {
query_id: info.query_id,
truncate: info.mode == spark_datasource::WriteMode::Truncate,
});
Ok(Write)
}
}
struct Scan;
impl ScanBuilder for Scan {
type Factory = BatchFactory;
fn plan(self) -> Result<ScanPlan<BatchFactory>> {
Ok(ScanPlan::new(BatchFactory(100), vec![5]))
}
}
/// Reads `factory + partition` from each partition of a batch scan.
#[derive(Serialize, Deserialize)]
struct BatchFactory(i64);
impl PartitionReaderFactory for BatchFactory {
type Partition = i64;
fn open<'task>(
&self,
partition: i64,
_: ReaderContext<'task>,
) -> Result<PartitionReader<'task>> {
Ok(Box::new(std::iter::once(Ok(batch(vec![
self.0 + partition,
])))))
}
}
struct Stream {
failure: String,
max_rows: Option<u64>,
}
/// The note checks that offsets stay on one line, as Spark requires.
#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
struct Offset {
position: u64,
note: String,
}
impl Offset {
fn at(position: u64) -> Self {
Self {
position,
note: "two\nlines".into(),
}
}
}
impl MicroBatchStream for Stream {
type Offset = Offset;
type Factory = StreamFactory;
fn initial_offset(&mut self) -> Result<Offset> {
Ok(Offset::at(0))
}
fn default_read_limit(&self) -> Result<ReadLimit> {
Ok(self
.max_rows
.map_or(ReadLimit::AllAvailable, ReadLimit::MaxRows))
}
fn latest_offset(&mut self, info: LatestOffsetInfo<Offset>) -> Result<Offset> {
if self.failure == "latest_panic" {
panic!("finding the latest offset panicked");
}
let start = info.start.position;
let run_end = info.run_end.map(|end| end.position);
record(Event::LatestOffset(start, info.limit, run_end));
let mut end = LATEST.load(Ordering::SeqCst);
if let ReadLimit::MaxRows(rows) = info.limit {
end = end.min(start + rows);
}
Ok(Offset::at(run_end.map_or(end, |run_end| end.min(run_end))))
}
fn plan(&mut self, info: MicroBatchInfo<Offset>) -> Result<ScanPlan<StreamFactory>> {
let positions = (info.start.position..info.end.position).collect();
Ok(ScanPlan::new(StreamFactory { scale: 10 }, positions))
}
fn commit(&mut self, info: CommitOffsetInfo<Offset>) -> Result<()> {
record(Event::OffsetCommitted(info.end.position));
Ok(())
}
}
/// Reads `scale * position` from the partition of each position of a micro-batch.
#[derive(Serialize, Deserialize)]
struct StreamFactory {
scale: i64,
}
impl PartitionReaderFactory for StreamFactory {
type Partition = u64;
fn open<'task>(
&self,
position: u64,
_: ReaderContext<'task>,
) -> Result<PartitionReader<'task>> {
let value = self.scale * position as i64;
Ok(Box::new(std::iter::once(Ok(batch(vec![value])))))
}
}
struct Write;
#[derive(Clone, Debug, PartialEq, Serialize, Deserialize)]
struct Message {
epoch_id: Option<i64>,
values: Vec<i64>,
}
impl StreamingWrite for Write {
type Factory = WriteFactory;
type Message = Message;
const SUPPORTS_TRUNCATE: bool = true;
fn new_writer_factory(&self) -> Result<WriteFactory> {
Ok(WriteFactory)
}
fn commit(&mut self, info: StreamingCommitInfo<Message>) -> Result<()> {
record(Event::EpochCommitted(info.epoch_id, info.messages));
Ok(())
}
fn abort(&mut self, info: StreamingAbortInfo<Message>) -> Result<()> {
record(Event::EpochAborted(info.epoch_id, info.messages));
Ok(())
}
}
impl Drop for Write {
fn drop(&mut self) {
record(Event::WriteDropped);
}
}
#[derive(Serialize, Deserialize)]
struct WriteFactory;
impl DataWriterFactory for WriteFactory {
type Message = Message;
fn open<'task>(
&self,
context: WriterContext<'task>,
) -> Result<PartitionWriter<'task, Message>> {
Ok(Box::new(Writer {
message: Message {
epoch_id: context.epoch_id,
values: vec![],
},
}))
}
}
struct Writer {
message: Message,
}
impl DataWriter for Writer {
type Message = Message;
fn write(&mut self, batch: RecordBatch) -> Result<()> {
let values = batch
.column(0)
.as_any()
.downcast_ref::<Int64Array>()
.unwrap();
self.message.values.extend(values.values());
Ok(())
}
fn commit(&mut self) -> Result<Message> {
Ok(self.message.clone())
}
fn abort(&mut self) -> Result<()> {
Ok(())
}
}
spark_datasource::export_data_source!(Source);
extern "C" {
fn spark_ds_init(host: *const SparkDsHost, library: *mut *const SparkDsLibrary) -> c_int;
}
fn library() -> &'static SparkDsLibrary {
// Spark's host structure stays valid until the JVM exits.
let host = Box::leak(Box::new(SparkDsHost {
size: size_of::<SparkDsHost>() as i64,
abi_version: ABI_VERSION,
spark_version: c"test".as_ptr(),
log: None,
is_log_enabled: None,
}));
let mut table = ptr::null();
assert_eq!(unsafe { spark_ds_init(host, &mut table) }, 0);
unsafe { &*table }
}
fn error(library: &SparkDsLibrary) -> String {
let message = unsafe { library.last_error.unwrap()() };
assert!(!message.is_null());
unsafe { CStr::from_ptr(message) }.to_str().unwrap().into()
}
fn success(library: &SparkDsLibrary, status: c_int) {
assert_eq!(status, 0, "{}", error(library));
}
/// The values that a call passes to its sinks: each sink gets a list of its own.
#[derive(Default)]
struct Sinks {
values: Vec<Vec<u8>>,
states: Vec<Vec<u8>>,
}
unsafe extern "C" fn add_value(sink: *mut c_void, bytes: *const u8, length: i64) {
let sinks = unsafe { &mut *sink.cast::<Sinks>() };
sinks
.values
.push(unsafe { std::slice::from_raw_parts(bytes, length as usize) }.to_vec());
}
unsafe extern "C" fn set_state(sink: *mut c_void, bytes: *const u8, length: i64) {
let sinks = unsafe { &mut *sink.cast::<Sinks>() };
sinks
.states
.push(unsafe { std::slice::from_raw_parts(bytes, length as usize) }.to_vec());
}
/// Calls a function of the table that passes its results to sinks.
fn emitted(library: &SparkDsLibrary, call: impl FnOnce(*mut c_void) -> c_int) -> (c_int, Sinks) {
let mut sinks = Sinks::default();
let status = call((&mut sinks as *mut Sinks).cast());
if status != 0 {
assert!(sinks.values.is_empty() && sinks.states.is_empty());
let _ = error(library);
}
(status, sinks)
}
/// The single value of a call, such as an offset.
fn single(library: &SparkDsLibrary, call: impl FnOnce(*mut c_void) -> c_int) -> Vec<u8> {
let (status, mut sinks) = emitted(library, call);
success(library, status);
assert_eq!(sinks.values.len(), 1);
assert!(sinks.states.is_empty());
sinks.values.pop().unwrap()
}
/// The partitions of a plan, and its state, which a planning call returns once.
fn planned(library: &SparkDsLibrary, call: impl FnOnce(*mut c_void) -> c_int) -> Sinks {
let (status, sinks) = emitted(library, call);
success(library, status);
assert_eq!(sinks.states.len(), 1, "a plan has a single state");
sinks
}
fn source(library: &SparkDsLibrary, options: &[(&str, &str)]) -> (*mut c_void, u64) {
let options: Vec<(CString, CString)> = options
.iter()
.map(|(key, value)| (CString::new(*key).unwrap(), CString::new(*value).unwrap()))
.collect();
let keys: Vec<*const c_char> = options.iter().map(|(key, _)| key.as_ptr()).collect();
let values: Vec<*const c_char> = options.iter().map(|(_, value)| value.as_ptr()).collect();
let info = SparkDsCreateInfo {
size: size_of::<SparkDsCreateInfo>() as i64,
name: c"streaming_test".as_ptr(),
option_keys: keys.as_ptr(),
option_values: values.as_ptr(),
num_options: keys.len() as i64,
};
let mut mask = 0;
let mut pointer = ptr::null_mut();
success(library, unsafe {
library.create.unwrap()(&info, &mut mask, &mut pointer)
});
(pointer, mask)
}
fn stream_reader(library: &SparkDsLibrary, options: &[(&str, &str)]) -> *mut c_void {
let (source, _) = source(library, options);
let mut schema = ArrowSchema::try_from(schema().as_ref()).unwrap();
let info = SparkDsStreamReaderInfo {
size: size_of::<SparkDsStreamReaderInfo>() as i64,
checkpoint_location: c"file:/tmp/checkpoint/sources/0".as_ptr(),
};
let mut reader = ptr::null_mut();
let status =
unsafe { library.create_stream_reader.unwrap()(source, &mut schema, &info, &mut reader) };
// Before close, which clears the error of the thread.
success(library, status);
// A stream reader outlives its source, which Spark closes right after creating it.
unsafe { library.close.unwrap()(source) };
reader
}
fn read_limit(kind: i32, max_rows: i64) -> SparkDsReadLimit {
SparkDsReadLimit {
size: size_of::<SparkDsReadLimit>() as i64,
kind,
max_rows,
}
}
fn latest(
library: &SparkDsLibrary,
reader: *mut c_void,
start: &CStr,
limit: &SparkDsReadLimit,
run_end: Option<&CStr>,
) -> (c_int, Sinks) {
let info = SparkDsLatestOffsetInfo {
size: size_of::<SparkDsLatestOffsetInfo>() as i64,
start: start.as_ptr(),
limit,
run_end: run_end.map_or(ptr::null(), CStr::as_ptr),
};
emitted(library, |sink| unsafe {
library.latest_offset.unwrap()(reader, &info, Some(add_value), sink)
})
}
fn stream_partitions(
library: &SparkDsLibrary,
reader: *mut c_void,
start: &CStr,
end: &CStr,
) -> (c_int, Sinks) {
let info = SparkDsStreamPartitionsInfo {
size: size_of::<SparkDsStreamPartitionsInfo>() as i64,
start: start.as_ptr(),
end: end.as_ptr(),
};
emitted(library, |sink| unsafe {
library.stream_partitions.unwrap()(reader, &info, Some(add_value), Some(set_state), sink)
})
}
fn task(partition: i32) -> SparkDsTaskContext {
SparkDsTaskContext {
size: size_of::<SparkDsTaskContext>() as i64,
stage_id: 1,
stage_attempt_number: 0,
partition_id: partition,
attempt_number: 0,
task_attempt_id: 10 + i64::from(partition),
is_interrupted: None,
}
}
/// Reads a partition with the read function that batch scans and micro-batches share.
fn read(library: &SparkDsLibrary, kind: i32, state: &[u8], partition: &[u8]) -> Vec<i64> {
let task = task(0);
let mut read_schema = ArrowSchema::try_from(schema().as_ref()).unwrap();
let info = SparkDsReadInfo {
size: size_of::<SparkDsReadInfo>() as i64,
kind,
schema: &mut read_schema,
reader_state: state.as_ptr(),
reader_state_length: state.len() as i64,
partition: partition.as_ptr(),
partition_length: partition.len() as i64,
};
let mut stream = FFI_ArrowArrayStream::empty();
success(library, unsafe {
library.read.unwrap()(&task, &info, &mut stream)
});
ArrowArrayStreamReader::try_new(stream)
.unwrap()
.flat_map(|batch| {
let batch = batch.unwrap();
let values = batch
.column(0)
.as_any()
.downcast_ref::<Int64Array>()
.unwrap();
values.values().to_vec()
})
.collect()
}
#[test]
fn streaming_operations_declare_their_capabilities_and_callbacks() {
let _guard = TEST_LOCK.lock().unwrap_or_else(PoisonError::into_inner);
let library = library();
assert!(library.create_reader.is_some() && library.create_stream_reader.is_some());
assert!(library.initial_offset.is_some() && library.latest_offset.is_some());
assert!(library.stream_partitions.is_some() && library.default_read_limit.is_some());
assert!(library.commit_offset.is_some() && library.close_stream_reader.is_some());
assert!(library.read.is_some() && library.create_writer.is_some());
assert!(library.create_data_writer.is_some() && library.close_data_writer.is_some());
// Every stream supports admission control, and only streaming writes truncate.
let streaming_truncate = abi::STREAMING_WRITE | abi::STREAMING_TRUNCATE;
let reads = abi::BATCH_READ | abi::MICRO_BATCH_READ | abi::ADMISSION_CONTROL;
for (capabilities, expected) in [
("", reads | streaming_truncate),
("append_only", reads | abi::STREAMING_WRITE),
// Truncation goes with the only write that supports it.
("no_streaming_write", reads),
] {
let (handle, mask) = source(library, &[("capabilities", capabilities)]);
assert_eq!(mask, expected, "{capabilities}");
unsafe { library.close.unwrap()(handle) };
}
}
#[test]
fn streams_plan_micro_batches_between_json_offsets() {
let _guard = TEST_LOCK.lock().unwrap_or_else(PoisonError::into_inner);
EVENTS.lock().unwrap().clear();
let library = library();
let reader = stream_reader(library, &[]);
let checkpoint = "file:/tmp/checkpoint/sources/0".to_owned();
assert_eq!(events(), vec![Event::StreamCreated(checkpoint.clone())]);
let mut limit = read_limit(abi::READ_MAX_ROWS, 99);
success(library, unsafe {
library.default_read_limit.unwrap()(reader, &mut limit)
});
assert_eq!(limit.kind, abi::READ_ALL_AVAILABLE, "no limit by default");
let initial_info = SparkDsInitialOffsetInfo {
size: size_of::<SparkDsInitialOffsetInfo>() as i64,
};
let initial = single(library, |sink| unsafe {
library.initial_offset.unwrap()(reader, &initial_info, Some(add_value), sink)
});
assert_eq!(initial, br#"{"position":0,"note":"two\nlines"}"#);
LATEST.store(3, Ordering::SeqCst);
let start = CString::new(initial).unwrap();
let all = read_limit(abi::READ_ALL_AVAILABLE, 0);
let (status, mut offsets) = latest(library, reader, &start, &all, None);
success(library, status);
let latest_offset = offsets.values.pop().unwrap();
assert!(
!latest_offset.contains(&b'\n'),
"an offset must be on one line"
);
let end = CString::new(latest_offset).unwrap();
let (status, plan) = stream_partitions(library, reader, &start, &end);
success(library, status);
assert_eq!(plan.states.len(), 1, "a plan has a single state");
let values: Vec<i64> = plan
.values
.iter()
.flat_map(|partition| read(library, abi::READ_MICRO_BATCH, &plan.states[0], partition))
.collect();
assert_eq!(values, vec![0, 10, 20]);
let commit = SparkDsCommitOffsetInfo {
size: size_of::<SparkDsCommitOffsetInfo>() as i64,
end: end.as_ptr(),
};
success(library, unsafe {
library.commit_offset.unwrap()(reader, &commit)
});
assert_eq!(
events(),
vec![
Event::StreamCreated(checkpoint),
Event::LatestOffset(0, ReadLimit::AllAvailable, None),
Event::OffsetCommitted(3)
]
);
// A batch scan of the same source reads its own partitions through the same function.
let (handle, _) = source(library, &[]);
let mut read_schema = ArrowSchema::try_from(schema().as_ref()).unwrap();
let reader_info = SparkDsReaderInfo {
size: size_of::<SparkDsReaderInfo>() as i64,
};
let mut planner = ptr::null_mut();
success(library, unsafe {
library.create_reader.unwrap()(handle, &mut read_schema, &reader_info, &mut planner)
});
unsafe { library.close.unwrap()(handle) };
let partitions_info = SparkDsPartitionsInfo {
size: size_of::<SparkDsPartitionsInfo>() as i64,
};
let batch_plan = planned(library, |sink| unsafe {
library.partitions.unwrap()(
planner,
&partitions_info,
Some(add_value),
Some(set_state),
sink,
)
});
unsafe { library.close_reader.unwrap()(planner) };
let batch_state = &batch_plan.states[0];
assert_eq!(
read(library, abi::READ_BATCH, batch_state, &batch_plan.values[0]),
vec![105]
);
// The kind of the read selects the factory: a batch state is not that of a micro-batch.
let task = task(0);
let mut stream = FFI_ArrowArrayStream::empty();
for kind in [abi::READ_MICRO_BATCH, 7] {
let mut read_schema = ArrowSchema::try_from(schema().as_ref()).unwrap();
let info = SparkDsReadInfo {
size: size_of::<SparkDsReadInfo>() as i64,
kind,
schema: &mut read_schema,
reader_state: batch_state.as_ptr(),
reader_state_length: batch_state.len() as i64,
partition: batch_plan.values[0].as_ptr(),
partition_length: batch_plan.values[0].len() as i64,
};
assert_ne!(
unsafe { library.read.unwrap()(&task, &info, &mut stream) },
0
);
assert!(stream.release.is_none());
}
assert!(error(library).contains("unsupported kind of read: 7"));
// An offset that does not decode fails the call, after which Spark closes the reader.
let invalid = c"not an offset";
let (status, _) = stream_partitions(library, reader, &start, invalid);
assert_ne!(status, 0);
let failure = error(library);
assert!(
failure.contains("cannot decode the offset not an offset"),
"{failure}"
);
unsafe { library.close_stream_reader.unwrap()(reader) };
}
#[test]
fn streams_limit_their_micro_batches_and_runs_with_admission_control() {
let _guard = TEST_LOCK.lock().unwrap_or_else(PoisonError::into_inner);
EVENTS.lock().unwrap().clear();
let library = library();
let reader = stream_reader(library, &[("maxrows", "2")]);
let mut limit = read_limit(abi::READ_ALL_AVAILABLE, 0);
success(library, unsafe {
library.default_read_limit.unwrap()(reader, &mut limit)
});
assert_eq!((limit.kind, limit.max_rows), (abi::READ_MAX_ROWS, 2));
LATEST.store(5, Ordering::SeqCst);
let start = CString::new(serde_json::to_string(&Offset::at(1)).unwrap()).unwrap();
let run_end = CString::new(serde_json::to_string(&Offset::at(2)).unwrap()).unwrap();
// The micro-batch of a limit, then that of a limit within the end of a run.
for (run_end, expected) in [(None, 3), (Some(run_end.as_c_str()), 2)] {
let (status, mut offsets) = latest(library, reader, &start, &limit, run_end);
success(library, status);
let offset: Offset = serde_json::from_slice(&offsets.values.pop().unwrap()).unwrap();
assert_eq!(offset, Offset::at(expected));
}
let unknown = read_limit(9, 0);
let (status, _) = latest(library, reader, &start, &unknown, None);
assert_ne!(status, 0);
assert!(error(library).contains("unsupported kind of read limit: 9"));
assert_eq!(
events()[1..],
[
Event::LatestOffset(1, ReadLimit::MaxRows(2), None),
Event::LatestOffset(1, ReadLimit::MaxRows(2), Some(2)),
]
);
unsafe { library.close_stream_reader.unwrap()(reader) };
}
#[test]
fn a_panic_is_the_failure_of_its_call() {
let _guard = TEST_LOCK.lock().unwrap_or_else(PoisonError::into_inner);
let library = library();
let reader = stream_reader(library, &[("failure", "latest_panic")]);
let start = CString::new(serde_json::to_string(&Offset::at(0)).unwrap()).unwrap();
let (status, _) = latest(library, reader, &start, &read_limit(0, 0), None);
assert_ne!(status, 0);
let failure = error(library);
assert!(
failure.contains("finding the latest offset panicked"),
"{failure}"
);
unsafe { library.close_stream_reader.unwrap()(reader) };
}
fn create_writer(
library: &SparkDsLibrary,
options: &[(&str, &str)],
mode: i32,
query: *const c_char,
) -> std::result::Result<(*mut c_void, Vec<u8>), String> {
let (source, _) = source(library, options);
let mut input = ArrowSchema::try_from(schema().as_ref()).unwrap();
let info = SparkDsWriteInfo {
size: size_of::<SparkDsWriteInfo>() as i64,
mode,
query_id: query,
};
let mut writer = ptr::null_mut();
let mut sinks = Sinks::default();
let status = unsafe {
library.create_writer.unwrap()(
source,
&mut input,
&info,
Some(set_state),
(&mut sinks as *mut Sinks).cast(),
&mut writer,
)
};
// Before close, which clears the error of the thread.
let result = if status == 0 {
assert_eq!(sinks.states.len(), 1, "a writer has a single state");
Ok((writer, sinks.states.pop().unwrap()))
} else {
assert!(writer.is_null() && sinks.states.is_empty());
Err(error(library))
};
unsafe { library.close.unwrap()(source) };
result
}
fn finish(
library: &SparkDsLibrary,
writer: *mut c_void,
abort: bool,
epoch_id: i64,
messages: &[Option<&[u8]>],
rejected: bool,
) -> c_int {
let pointers: Vec<_> = messages
.iter()
.map(|message| message.map_or(ptr::null(), |bytes| bytes.as_ptr()))
.collect();
let lengths: Vec<_> = messages
.iter()
.map(|message| message.map_or(0, |bytes| bytes.len() as i64))
.collect();
let info = SparkDsFinishWriteInfo {
size: size_of::<SparkDsFinishWriteInfo>() as i64,
epoch_id,
messages: pointers.as_ptr(),
message_lengths: lengths.as_ptr(),
num_messages: messages.len() as i64,
rejected: i64::from(rejected),
};
let callback = if abort {
library.abort.unwrap()
} else {
library.commit.unwrap()
};
unsafe { callback(writer, &info) }
}
#[test]
fn streaming_writes_pass_the_query_and_the_epoch_to_the_write_and_its_tasks() {
let _guard = TEST_LOCK.lock().unwrap_or_else(PoisonError::into_inner);
EVENTS.lock().unwrap().clear();
let library = library();
let (writer, state) = create_writer(library, &[], 1, c"query-1".as_ptr()).unwrap();
// Each task gets the epoch of its micro-batch, and the schema of the data.
let task = task(0);
let mut data_writer = ptr::null_mut();
let create = |epoch_id: i64, data_writer: &mut *mut c_void| {
let mut input = ArrowSchema::try_from(schema().as_ref()).unwrap();
let info = SparkDsDataWriterInfo {
size: size_of::<SparkDsDataWriterInfo>() as i64,
schema: &mut input,
writer_state: state.as_ptr(),
writer_state_length: state.len() as i64,
epoch_id,
};
unsafe { library.create_data_writer.unwrap()(&task, &info, data_writer) }
};
assert_ne!(create(-2, &mut data_writer), 0);
assert!(error(library).contains("invalid epoch ID -2"));
// -1 is the epoch of a batch write, which this source does not have.
assert_ne!(create(-1, &mut data_writer), 0);
assert!(data_writer.is_null());
success(library, create(7, &mut data_writer));
let mut array = FFI_ArrowArray::new(&StructArray::from(batch(vec![4, 2])).to_data());
success(library, unsafe {
library.write.unwrap()(data_writer, &mut array)
});
let finish_info = SparkDsFinishDataWriterInfo {
size: size_of::<SparkDsFinishDataWriterInfo>() as i64,
};
let message = single(library, |sink| unsafe {
library.commit_data_writer.unwrap()(data_writer, &finish_info, Some(add_value), sink)
});
unsafe { library.close_data_writer.unwrap()(data_writer) };
success(
library,
finish(library, writer, false, 7, &[Some(&message)], false),
);
// Spark closes the writer of a micro-batch, which drops its write, once it committed or
// aborted the micro-batch, before it creates the writer of the next one.
unsafe { library.close_writer.unwrap()(writer) };
let (writer, _) = create_writer(library, &[], 0, c"query-1".as_ptr()).unwrap();
success(library, finish(library, writer, true, 8, &[None], false));
unsafe { library.close_writer.unwrap()(writer) };
// A rejected writer is dropped without its abort at close, as when its creation fails.
let (writer, _) = create_writer(library, &[], 0, c"query-1".as_ptr()).unwrap();
success(library, finish(library, writer, true, -1, &[], true));
unsafe { library.close_writer.unwrap()(writer) };
// The source has no batch writes, and an append-only source does not truncate.
let error = create_writer(library, &[], 0, ptr::null()).unwrap_err();
assert!(error.contains("does not support batch writes"), "{error}");
let options = [("capabilities", "append_only")];
let error = create_writer(library, &options, 1, c"query-2".as_ptr()).unwrap_err();
assert!(
error.contains("does not support the output mode complete"),
"{error}"
);
let committed = Message {
epoch_id: Some(7),
values: vec![4, 2],
};
let created = |truncate| Event::WriteCreated {
query_id: "query-1".into(),
truncate,
};
assert_eq!(
events(),
vec![
created(true),
Event::EpochCommitted(7, vec![committed]),
Event::WriteDropped,
created(false),
Event::EpochAborted(8, vec![None]),
Event::WriteDropped,
created(false),
Event::WriteDropped,
]
);
}