blob: 45fa04f060d0ffe103cbcc6be0a8669641e295cb [file]
// Licensed to the Apache Software Foundation (ASF) under one or more
// contributor license agreements. See the NOTICE file distributed with
// this work for additional information regarding copyright ownership.
// The ASF licenses this file to You under the Apache License, Version 2.0
// (the "License"); you may not use this file except in compliance with
// the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
use std::ffi::{c_char, c_int, c_void, CString};
use std::sync::Arc;
use arrow_array::ffi::FFI_ArrowArray;
use arrow_array::ffi_stream::FFI_ArrowArrayStream;
use arrow_array::types::{Decimal128Type, DecimalType};
use arrow_array::{Array, RecordBatch, StructArray};
use arrow_data::ArrayData;
use arrow_schema::{ffi::FFI_ArrowSchema, DataType, Field, Fields, Schema, SchemaRef, TimeUnit};
use super::{c_message, drop_provider, provider_call};
use crate::{PartitionReader, Result};
struct StreamState<'task> {
reader: Option<PartitionReader<'task>>,
// The schema to read, whose columns, Spark types, and nullability each batch must have.
expected: SchemaRef,
// The schema of the stream: that of the first batch, which every batch must have, or
// the schema to read of a reader without batches. Unknown until either is read.
schema: Option<SchemaRef>,
// The schema of the last batch that the reader returned, with the indices of the columns
// to read in it, or None when it has exactly those columns.
projection: Option<(SchemaRef, Option<Vec<usize>>)>,
// The first batch, when get_schema read it to learn the schema.
pending: Option<ArrayData>,
last_error: Option<CString>,
finished: bool,
}
impl StreamState<'_> {
/// Reads and checks the next batch. The first batch, or the end of the reader,
/// determines the schema of the stream.
fn read_next(&mut self) -> Result<Option<ArrayData>> {
let Some(batch) = self.reader.as_mut().expect("reader is present").next() else {
self.finished = true;
self.schema.get_or_insert_with(|| self.expected.clone());
return Ok(None);
};
let batch = self.project(batch?)?;
match &self.schema {
Some(schema) if batch.schema() != *schema => {
return Err(schema_change(schema, &batch.schema()).into());
}
Some(_) => {}
None => {
if canonical_fields(batch.schema().fields())?
!= canonical_fields(self.expected.fields())?
{
return Err("executor reader schema does not match the schema to read".into());
}
self.schema = Some(batch.schema());
}
}
let data = StructArray::from(batch).to_data();
validate_layout(&data)?;
for (column, field) in data.child_data().iter().zip(self.expected.fields()) {
check_values(column, field, field.name(), None)?;
}
Ok(Some(data))
}
/// Keeps the columns of the schema to read, in its order, from a batch that may have more,
/// such as all the columns of the source. Resolves the columns once for each schema.
fn project(&mut self, batch: RecordBatch) -> Result<RecordBatch> {
let schema = batch.schema();
if !matches!(&self.projection, Some((last, _)) if *last == schema) {
self.projection = Some((schema.clone(), projection(&schema, &self.expected)?));
}
match &self.projection {
Some((_, Some(indices))) => Ok(batch.project(indices)?),
_ => Ok(batch),
}
}
}
/// Describes how the schema of a batch differs from that of the stream. Both have the
/// columns of the schema to read, so either a column or the metadata of the schema differs.
fn schema_change(stream: &Schema, batch: &Schema) -> String {
let mut fields = stream.fields().iter().zip(batch.fields());
match fields.find(|(first, field)| first != field) {
Some((first, field)) => format!(
"reader returned a batch with a different schema: the column {} is {first:?} in \
the first batch and {field:?} in this one",
first.name()
),
None => "reader returned a batch with a different schema: its metadata differs".to_owned(),
}
}
/// The indices of the columns of the schema to read, in its order, in the schema of a batch
/// that may have more, or None when the batch has exactly those columns.
fn projection(schema: &Schema, expected: &Schema) -> Result<Option<Vec<usize>>> {
let names = |fields: &Fields| fields.iter().map(|field| field.name().clone()).collect();
let actual: Vec<String> = names(schema.fields());
let wanted: Vec<String> = names(expected.fields());
if actual == wanted {
return Ok(None);
}
let indices =
wanted
.iter()
.map(|name| {
actual.iter().position(|actual| actual == name).ok_or_else(|| {
format!("reader returned a batch without the column {name} of the schema to read")
})
})
.collect::<std::result::Result<Vec<_>, _>>()?;
Ok(Some(indices))
}
impl Drop for StreamState<'_> {
fn drop(&mut self) {
// release has no error return channel, but it must never unwind into C.
if let Some(reader) = self.reader.take() {
drop_provider(reader);
}
}
}
/// The reader may borrow the host task context until the consumer releases
/// this stream. Erasure into C private_data is confined to this boundary.
///
/// # Safety
/// The caller must keep everything borrowed by the reader alive until release,
/// and the consumer must obey the Arrow C Stream ownership protocol.
pub(super) unsafe fn export(
reader: PartitionReader<'_>,
expected_schema: &SchemaRef,
) -> FFI_ArrowArrayStream {
let state = Box::new(StreamState {
reader: Some(reader),
expected: expected_schema.clone(),
schema: None,
projection: None,
pending: None,
last_error: None,
finished: false,
});
FFI_ArrowArrayStream {
get_schema: Some(get_schema),
get_next: Some(get_next),
get_last_error: Some(get_last_error),
release: Some(release),
private_data: Box::into_raw(state).cast::<c_void>(),
}
}
fn callback(
state: &mut StreamState<'_>,
action: impl FnOnce(&mut StreamState<'_>) -> Result<()>,
) -> c_int {
// A failed stream stays failed, with its error: an engine that calls it again must not
// read a failed partition as complete.
if state.last_error.is_some() {
return 5;
}
let Err(message) = provider_call(|| action(state)) else {
return 0;
};
state.last_error = Some(c_message(message));
5 // EIO; get_last_error supplies the provider's actual error message.
}
unsafe extern "C" fn get_schema(
stream: *mut FFI_ArrowArrayStream,
output: *mut FFI_ArrowSchema,
) -> c_int {
let state = unsafe { &mut *(*stream).private_data.cast::<StreamState<'_>>() };
callback(state, |state| {
// The reader does not declare a schema: read the first batch, which has it.
if state.schema.is_none() && !state.finished {
state.pending = state.read_next()?;
}
let schema = state.schema.as_ref().unwrap_or(&state.expected);
let schema = FFI_ArrowSchema::try_from(schema.as_ref())?;
unsafe { output.write(schema) };
Ok(())
})
}
unsafe extern "C" fn get_next(
stream: *mut FFI_ArrowArrayStream,
output: *mut FFI_ArrowArray,
) -> c_int {
let state = unsafe { &mut *(*stream).private_data.cast::<StreamState<'_>>() };
callback(state, |state| {
let data = match state.pending.take() {
Some(data) => Some(data),
None if state.finished => None,
None => state.read_next()?,
};
match data {
Some(data) => unsafe { output.write(FFI_ArrowArray::new(&data)) },
// A null release callback denotes EOF, including for zero-column data.
None => unsafe { output.write(FFI_ArrowArray::empty()) },
}
Ok(())
})
}
unsafe extern "C" fn get_last_error(stream: *mut FFI_ArrowArrayStream) -> *const c_char {
let state = unsafe { &*(*stream).private_data.cast::<StreamState<'_>>() };
state
.last_error
.as_ref()
.map_or(std::ptr::null(), |error| error.as_ptr())
}
unsafe extern "C" fn release(stream: *mut FFI_ArrowArrayStream) {
let state = unsafe { (*stream).private_data.cast::<StreamState<'_>>() };
// Mark released before destruction. The consumer owns the stream structure.
unsafe { stream.write(FFI_ArrowArrayStream::empty()) };
drop_provider(unsafe { Box::from_raw(state) });
}
// Compare the Spark logical types of version 1 of the ABI, ignoring Arrow
// nullability/metadata and the interchangeable physical encodings of strings and
// binary data. Spark also distinguishes only the presence of a time zone, not its
// spelling. Nullability is checked against the data instead, in check_values.
pub(crate) fn validate_schema(schema: &SchemaRef) -> Result<()> {
canonical_fields(schema.fields())?;
Ok(())
}
pub(crate) fn canonical_fields(fields: &Fields) -> Result<Fields> {
fields
.iter()
.map(|field| {
Ok(Arc::new(Field::new(
field.name(),
canonical_type(field.data_type())?,
false,
)))
})
.collect()
}
fn canonical_type(data_type: &DataType) -> Result<DataType> {
use DataType::*;
Ok(match data_type {
Boolean | Int8 | Int16 | Int32 | Int64 | Float32 | Float64 | Date32 => data_type.clone(),
Utf8 | LargeUtf8 | Utf8View => Utf8,
Binary | LargeBinary | BinaryView => Binary,
Decimal128(precision, _) if *precision > 0 && *precision <= 38 => data_type.clone(),
Timestamp(TimeUnit::Microsecond, tz) => {
Timestamp(TimeUnit::Microsecond, tz.as_ref().map(|_| "UTC".into()))
}
Struct(fields) => Struct(canonical_fields(fields)?),
List(field) => List(Arc::new(Field::new(
"item",
canonical_type(field.data_type())?,
false,
))),
Map(entries, _) => {
let Struct(fields) = entries.data_type() else {
return Err("Arrow map entries must be a struct".into());
};
if fields.len() != 2 {
return Err("Arrow map entries must have a key and a value".into());
}
let entries = Struct(
vec![
Field::new("key", canonical_type(fields[0].data_type())?, false),
Field::new("value", canonical_type(fields[1].data_type())?, false),
]
.into(),
);
Map(Arc::new(Field::new("entries", entries, false)), false)
}
_ => return Err(format!("unsupported Spark Arrow type: {data_type}").into()),
})
}
/// Rejects a null where the schema to read declares a column, a field, an element, or a map
/// value not nullable, unless its parent is null, as Spark does after reading, and a decimal
/// value that does not fit in the precision of its type, as Spark does when it converts the
/// value to a row. `referenced` tells which values belong to a parent value that is not null.
fn check_values(
data: &ArrayData,
field: &Field,
column: &str,
referenced: Option<&[bool]>,
) -> Result<()> {
let is_referenced = |index: usize| referenced.is_none_or(|mask| mask[index]);
if !field.is_nullable()
&& data.null_count() > 0
&& (0..data.len()).any(|index| data.is_null(index) && is_referenced(index))
{
return Err(format!(
"The column {column} has a null, but the schema to read declares it not nullable"
)
.into());
}
let valid = |index: usize| data.is_valid(index) && is_referenced(index);
// The mask of a container is built only when a child has values to check.
match field.data_type() {
DataType::Decimal128(precision, scale) => {
let values = data.buffer::<i128>(0);
for index in (0..data.len()).filter(|index| valid(*index)) {
let value = values[index];
if Decimal128Type::validate_decimal_precision(value, *precision, *scale).is_err() {
return Err(format!(
"The column {column} has the value {}, which does not fit in the \
precision {precision} of its decimal type in the schema to read",
Decimal128Type::format_decimal(
value,
Decimal128Type::MAX_PRECISION,
*scale
)
)
.into());
}
}
}
DataType::Struct(fields) => {
let children: Vec<_> = data
.child_data()
.iter()
.zip(fields)
.filter(|(child, field)| has_checks(child, field))
.collect();
if !children.is_empty() {
let mask: Vec<bool> = (0..data.len()).map(valid).collect();
for (child, field) in children {
let name = format!("{column}.{}", field.name());
check_values(child, field, &name, Some(&mask))?;
}
}
}
DataType::List(element) => {
let child = &data.child_data()[0];
if has_checks(child, element) {
let mask = elements(data, valid)?;
let name = format!("{column}.element");
check_values(child, element, &name, Some(&mask))?;
}
}
DataType::Map(entries, _) => {
let DataType::Struct(fields) = entries.data_type() else {
return Err("Arrow map entries must be a struct".into());
};
let entries = &data.child_data()[0];
// Spark does not allow null keys.
let key = fields[0].as_ref().clone().with_nullable(false);
let children: Vec<_> = [
(&entries.child_data()[0], &key, "key"),
(&entries.child_data()[1], fields[1].as_ref(), "value"),
]
.into_iter()
.filter(|(child, field, _)| has_checks(child, field))
.collect();
if !children.is_empty() {
let mask = elements(data, valid)?;
for (child, field, part) in children {
let name = format!("{column}.{part}");
check_values(child, field, &name, Some(&mask))?;
}
}
}
_ => {}
}
Ok(())
}
/// Whether `data` has values that `check_values` must check, whatever their parents: a null
/// where `field` or a field, an element, a map key, or a map value under it is not nullable,
/// or a decimal. Null counts are cached, so this reads no values.
fn has_checks(data: &ArrayData, field: &Field) -> bool {
if !field.is_nullable() && data.null_count() > 0 {
return true;
}
match field.data_type() {
DataType::Decimal128(..) => true,
DataType::Struct(fields) => data
.child_data()
.iter()
.zip(fields)
.any(|(child, field)| has_checks(child, field)),
DataType::List(element) => has_checks(&data.child_data()[0], element),
DataType::Map(entries, _) => {
let DataType::Struct(fields) = entries.data_type() else {
return true;
};
let entries = data.child_data()[0].child_data();
// Keys are never null, whatever their field declares.
entries[0].null_count() > 0
|| has_checks(&entries[0], &fields[0])
|| has_checks(&entries[1], &fields[1])
}
_ => false,
}
}
/// Whether each element of a list or map belongs to a value that `valid` accepts.
fn elements(data: &ArrayData, valid: impl Fn(usize) -> bool) -> Result<Vec<bool>> {
let offsets = data.buffer::<i32>(0);
let mut mask = vec![false; data.child_data()[0].len()];
for index in 0..data.len() {
if valid(index) {
let start = usize::try_from(offsets[index])?;
let end = usize::try_from(offsets[index + 1])?;
mask.get_mut(start..end)
.ok_or("Arrow offsets exceed the length of their values")?
.fill(true);
}
}
Ok(mask)
}
fn validate_layout(data: &ArrayData) -> Result<()> {
if data.offset() != 0 {
return Err(format!(
"Spark requires zero Arrow array offsets: {} has offset {}; normalize sliced arrays",
data.data_type(),
data.offset(),
)
.into());
}
for child in data.child_data() {
validate_layout(child)?;
}
Ok(())
}
#[cfg(test)]
mod tests {
use std::ffi::CStr;
use std::sync::atomic::{AtomicUsize, Ordering};
use arrow_array::builder::{
BooleanBuilder, Decimal128Builder, Int32Builder, ListBuilder, MapBuilder,
NullBufferBuilder, OffsetBufferBuilder, StringBuilder, StructBuilder,
};
use arrow_array::ffi_stream::ArrowArrayStreamReader;
use arrow_array::{
ArrayRef, BooleanArray, Decimal128Array, Int32Array, Int64Array, LargeStringArray,
ListArray, MapArray, RecordBatch, RecordBatchReader, StringArray,
};
use arrow_schema::{ArrowError, DataType, Field, Schema};
use super::*;
struct Reader {
batches: std::vec::IntoIter<std::result::Result<RecordBatch, ArrowError>>,
panic_next: bool,
panic_drop: bool,
drops: Arc<AtomicUsize>,
}
impl Iterator for Reader {
type Item = std::result::Result<RecordBatch, ArrowError>;
fn next(&mut self) -> Option<Self::Item> {
assert!(!self.panic_next, "iterator failed");
self.batches.next()
}
}
impl Drop for Reader {
fn drop(&mut self) {
self.drops.fetch_add(1, Ordering::SeqCst);
assert!(!self.panic_drop, "drop failed");
}
}
fn schema() -> SchemaRef {
Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]))
}
fn export_with(reader: Reader, expected: &SchemaRef) -> FFI_ArrowArrayStream {
// Test readers own all their state and cannot borrow a host context.
unsafe { super::export(Box::new(reader), expected) }
}
fn export(reader: Reader) -> FFI_ArrowArrayStream {
export_with(reader, &schema())
}
fn reader_of(batches: Vec<RecordBatch>) -> Reader {
Reader {
batches: batches.into_iter().map(Ok).collect::<Vec<_>>().into_iter(),
panic_next: false,
panic_drop: false,
drops: Arc::default(),
}
}
fn reader() -> Reader {
let values = Arc::new(Int64Array::from(vec![3, 5]));
reader_of(vec![RecordBatch::try_new(schema(), vec![values]).unwrap()])
}
/// Calls get_schema, which reads the first batch, and returns its error, if any.
fn first_batch_error(stream: &mut FFI_ArrowArrayStream) -> Option<String> {
let mut schema = FFI_ArrowSchema::empty();
if unsafe { stream.get_schema.unwrap()(stream, &mut schema) } == 0 {
return None;
}
let error = unsafe { stream.get_last_error.unwrap()(stream) };
Some(
unsafe { CStr::from_ptr(error) }
.to_str()
.unwrap()
.to_owned(),
)
}
fn batch_error(batch: RecordBatch, expected: &SchemaRef) -> Option<String> {
first_batch_error(&mut export_with(reader_of(vec![batch]), expected))
}
fn at_end(stream: &mut FFI_ArrowArrayStream) -> bool {
let mut array = FFI_ArrowArray::empty();
let status = unsafe { stream.get_next.unwrap()(stream, &mut array) };
status == 0 && array.is_released()
}
/// Checks that a failed stream fails again, with the same error, instead of reporting the
/// end of its batches.
fn stays_failed(stream: &mut FFI_ArrowArrayStream, error: &str) {
let mut array = FFI_ArrowArray::empty();
assert_ne!(unsafe { stream.get_next.unwrap()(stream, &mut array) }, 0);
let mut schema = FFI_ArrowSchema::empty();
assert_ne!(
unsafe { stream.get_schema.unwrap()(stream, &mut schema) },
0
);
let again = unsafe { CStr::from_ptr(stream.get_last_error.unwrap()(stream)) };
assert_eq!(again.to_str().unwrap(), error);
}
#[test]
fn imported_batch_survives_early_stream_release() {
let reader = reader();
let drops = reader.drops.clone();
let mut stream = ArrowArrayStreamReader::try_new(export(reader)).unwrap();
let batch = stream.next().unwrap().unwrap();
drop(stream);
assert_eq!(drops.load(Ordering::SeqCst), 1);
assert_eq!(batch.num_rows(), 2);
let values = batch
.column(0)
.as_any()
.downcast_ref::<Int64Array>()
.unwrap();
assert_eq!(values.values(), &[3, 5]);
}
#[test]
fn empty_batch_is_not_eof() {
let mut reader = reader();
let first = reader.batches.next().unwrap();
reader.batches = vec![Ok(RecordBatch::new_empty(schema())), first].into_iter();
let mut stream = ArrowArrayStreamReader::try_new(export(reader)).unwrap();
assert_eq!(stream.next().unwrap().unwrap().num_rows(), 0);
assert_eq!(stream.next().unwrap().unwrap().num_rows(), 2);
assert!(stream.next().is_none());
}
#[test]
fn a_reader_without_batches_has_the_schema_to_read() {
let mut stream = ArrowArrayStreamReader::try_new(export(reader_of(vec![]))).unwrap();
assert_eq!(stream.schema(), schema());
assert!(stream.next().is_none());
}
#[test]
fn the_first_batch_determines_the_arrow_types_of_the_stream() {
// The schema to read has a Utf8 string, which the reader returns as LargeUtf8.
let expected = Arc::new(Schema::new(vec![Field::new("label", DataType::Utf8, true)]));
let actual = Arc::new(Schema::new(vec![Field::new(
"label",
DataType::LargeUtf8,
true,
)]));
let labels = Arc::new(LargeStringArray::from(vec!["a", "b"]));
let batch = RecordBatch::try_new(actual.clone(), vec![labels]).unwrap();
let exported = export_with(reader_of(vec![batch]), &expected);
let mut stream = ArrowArrayStreamReader::try_new(exported).unwrap();
assert_eq!(stream.schema(), actual);
let batch = stream.next().unwrap().unwrap();
let labels = batch
.column(0)
.as_any()
.downcast_ref::<LargeStringArray>()
.unwrap();
assert_eq!(labels.value(1), "b");
assert!(stream.next().is_none());
}
#[test]
fn the_sdk_keeps_the_columns_of_the_schema_to_read_by_name() {
// The reader ignores pruning and returns all the columns of the source, in its order.
let all = Arc::new(Schema::new(vec![
Field::new("id", DataType::Int64, false),
Field::new("flag", DataType::Boolean, false),
]));
let ids = Arc::new(Int64Array::from(vec![3, 5]));
let flags = Arc::new(BooleanArray::from(vec![true, false]));
let batch = RecordBatch::try_new(all, vec![ids, flags]).unwrap();
for (columns, rows) in [(vec!["flag", "id"], 2), (vec![], 2)] {
let fields: Vec<_> = columns
.iter()
.map(|name| match *name {
"id" => Field::new("id", DataType::Int64, false),
_ => Field::new("flag", DataType::Boolean, false),
})
.collect();
let expected = Arc::new(Schema::new(fields));
let exported = export_with(reader_of(vec![batch.clone()]), &expected);
let mut stream = ArrowArrayStreamReader::try_new(exported).unwrap();
assert_eq!(stream.schema(), expected);
let read = stream.next().unwrap().unwrap();
assert_eq!(read.schema(), expected);
assert_eq!(read.num_rows(), rows, "{columns:?}");
assert!(stream.next().is_none());
}
}
#[test]
fn the_sdk_resolves_the_columns_again_when_a_batch_reorders_them() {
let id = Field::new("id", DataType::Int64, false);
let flag = Field::new("flag", DataType::Boolean, false);
let flags = || Arc::new(BooleanArray::from(vec![true, false]));
let first = RecordBatch::try_new(
Arc::new(Schema::new(vec![id.clone(), flag.clone()])),
vec![Arc::new(Int64Array::from(vec![3, 5])), flags()],
)
.unwrap();
let second = RecordBatch::try_new(
Arc::new(Schema::new(vec![flag, id.clone()])),
vec![flags(), Arc::new(Int64Array::from(vec![7, 9]))],
)
.unwrap();
let expected = Arc::new(Schema::new(vec![id]));
let exported = export_with(reader_of(vec![first, second]), &expected);
let mut stream = ArrowArrayStreamReader::try_new(exported).unwrap();
for ids in [vec![3, 5], vec![7, 9]] {
let read = stream.next().unwrap().unwrap();
assert_eq!(read.schema(), expected);
let column = read.column(0).as_any().downcast_ref::<Int64Array>();
assert_eq!(column, Some(&Int64Array::from(ids)));
}
assert!(stream.next().is_none());
}
#[test]
fn get_next_may_read_the_first_batch_before_get_schema() {
let mut stream = export(reader());
let mut array = FFI_ArrowArray::empty();
assert_eq!(
unsafe { stream.get_next.unwrap()(&mut stream, &mut array) },
0
);
assert_eq!(array.len(), 2);
let mut exported = FFI_ArrowSchema::empty();
assert_eq!(
unsafe { stream.get_schema.unwrap()(&mut stream, &mut exported) },
0
);
assert_eq!(Schema::try_from(&exported).unwrap(), *schema());
assert!(at_end(&mut stream));
}
#[test]
fn iterator_panics_and_nul_errors_become_stream_errors() {
let mut panicking = reader();
panicking.panic_next = true;
let error = first_batch_error(&mut export(panicking)).unwrap();
assert!(error.contains("iterator failed"), "{error}");
let mut failing = reader();
failing.batches = vec![Err(ArrowError::ComputeError("bad\0value".into()))].into_iter();
let error = first_batch_error(&mut export(failing)).unwrap();
assert!(error.contains("bad\\0value"), "{error}");
// The error of a later batch is reported by get_next instead.
let mut failing = reader();
let first = failing.batches.next().unwrap();
let late = Err(ArrowError::ComputeError("late failure".into()));
failing.batches = vec![first, late].into_iter();
let mut stream = ArrowArrayStreamReader::try_new(export(failing)).unwrap();
assert_eq!(stream.next().unwrap().unwrap().num_rows(), 2);
let error = stream.next().unwrap().unwrap_err().to_string();
assert!(error.contains("late failure"), "{error}");
// The stream stays failed, with the same error.
assert_eq!(stream.next().unwrap().unwrap_err().to_string(), error);
}
#[test]
fn iterator_and_drop_panics_do_not_escape_the_stream_boundary() {
let mut failing = reader();
let drops = failing.drops.clone();
failing.panic_next = true;
failing.panic_drop = true;
let error = first_batch_error(&mut export(failing)).unwrap();
assert!(error.contains("iterator failed"), "{error}");
assert_eq!(drops.load(Ordering::SeqCst), 1);
let mut failing = reader();
let drops = failing.drops.clone();
failing.panic_drop = true;
drop(export(failing));
assert_eq!(drops.load(Ordering::SeqCst), 1);
}
#[test]
fn a_schema_mismatch_drops_the_reader() {
let reader = reader();
let drops = reader.drops.clone();
let expected = Arc::new(Schema::new(vec![Field::new("id", DataType::Utf8, false)]));
let mut stream = export_with(reader, &expected);
let error = first_batch_error(&mut stream).unwrap();
assert!(
error.contains("does not match the schema to read"),
"{error}"
);
stays_failed(&mut stream, &error);
drop(stream);
assert_eq!(drops.load(Ordering::SeqCst), 1);
}
#[test]
fn logical_schema_comparison_accepts_equivalent_arrow_representations() {
let actual = Schema::new(vec![
Field::new("text", DataType::LargeUtf8, true),
Field::new("bytes", DataType::BinaryView, true),
Field::new(
"time",
DataType::Timestamp(TimeUnit::Microsecond, Some("Europe/Paris".into())),
true,
),
Field::new(
"nested",
DataType::Struct(vec![Field::new("text", DataType::Utf8View, true)].into()),
true,
),
]);
let expected = Schema::new(vec![
Field::new("text", DataType::Utf8, false),
Field::new("bytes", DataType::Binary, false),
Field::new(
"time",
DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into())),
false,
),
Field::new(
"nested",
DataType::Struct(vec![Field::new("text", DataType::Utf8, false)].into()),
false,
),
]);
assert_eq!(
canonical_fields(actual.fields()).unwrap(),
canonical_fields(expected.fields()).unwrap()
);
// A timestamp without a time zone remains a distinct Spark logical type.
assert_ne!(
canonical_type(&DataType::Timestamp(TimeUnit::Microsecond, None)).unwrap(),
canonical_type(&DataType::Timestamp(
TimeUnit::Microsecond,
Some("UTC".into())
))
.unwrap()
);
}
#[test]
fn unsupported_types_are_rejected_inside_nested_fields() {
let nested_types = [
DataType::List(Arc::new(Field::new("item", DataType::UInt64, false))),
DataType::Struct(
vec![Field::new(
"dictionary",
DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)),
false,
)]
.into(),
),
DataType::Map(
Arc::new(Field::new(
"entries",
DataType::Struct(
vec![
Field::new("key", DataType::Utf8, false),
Field::new("value", DataType::Decimal256(40, 2), true),
]
.into(),
),
false,
)),
false,
),
];
for data_type in nested_types {
let schema = Arc::new(Schema::new(vec![Field::new("nested", data_type, true)]));
assert!(validate_schema(&schema)
.unwrap_err()
.to_string()
.contains("unsupported Spark Arrow type"));
}
}
#[test]
fn a_valid_sliced_boolean_inside_a_struct_is_rejected_before_export() {
let values = BooleanArray::from(vec![true, false, true]).slice(1, 2);
assert_eq!(values.to_data().offset(), 1);
let fields: Fields = vec![Field::new("flag", DataType::Boolean, false)].into();
let nested = StructArray::new(fields.clone(), vec![Arc::new(values)], None);
let schema = Arc::new(Schema::new(vec![Field::new(
"nested",
DataType::Struct(fields),
false,
)]));
let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(nested)]).unwrap();
let reader = reader_of(vec![batch]);
let drops = reader.drops.clone();
// All arrays are valid, owned Arrow values, without a borrowed task context.
let mut stream = export_with(reader, &schema);
let error = first_batch_error(&mut stream).unwrap();
assert!(error.contains("zero Arrow array offsets"), "{error}");
assert!(error.contains("offset 1"), "{error}");
stays_failed(&mut stream, &error);
drop(stream);
assert_eq!(drops.load(Ordering::SeqCst), 1);
}
#[test]
fn intervals_and_the_null_type_are_not_version_1_types() {
for data_type in [
DataType::Null,
DataType::Duration(TimeUnit::Microsecond),
DataType::Interval(arrow_schema::IntervalUnit::YearMonth),
DataType::Interval(arrow_schema::IntervalUnit::MonthDayNano),
] {
let schema = Arc::new(Schema::new(vec![Field::new("value", data_type, true)]));
assert!(validate_schema(&schema).is_err());
}
}
#[test]
fn nulls_must_respect_the_nullability_of_the_schema_to_read() {
let values = Arc::new(Int64Array::from(vec![Some(1), None]));
let nullable = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, true)]));
let required = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, false)]));
let batch = RecordBatch::try_new(nullable.clone(), vec![values]).unwrap();
let error = batch_error(batch.clone(), &required);
assert!(error.unwrap().contains("The column id has a null"));
assert_eq!(batch_error(batch, &nullable), None);
}
#[test]
fn nested_nulls_are_allowed_only_under_a_null_parent() {
let required = Fields::from(vec![Field::new("flag", DataType::Boolean, false)]);
let nullable = Fields::from(vec![Field::new("flag", DataType::Boolean, true)]);
let schema = |fields: &Fields| {
Arc::new(Schema::new(vec![Field::new(
"nested",
DataType::Struct(fields.clone()),
true,
)]))
};
fn flags(builder: &mut StructBuilder) -> &mut BooleanBuilder {
builder.field_builder::<BooleanBuilder>(0).unwrap()
}
for (parent_is_valid, rejected) in [(false, false), (true, true)] {
let mut builder =
StructBuilder::new(nullable.clone(), vec![Box::new(BooleanBuilder::new())]);
flags(&mut builder).append_value(true);
builder.append(true);
flags(&mut builder).append_null();
builder.append(parent_is_valid);
let batch =
RecordBatch::try_new(schema(&nullable), vec![Arc::new(builder.finish())]).unwrap();
let error = batch_error(batch, &schema(&required));
assert_eq!(error.is_some(), rejected, "{error:?}");
if let Some(error) = error {
assert!(
error.contains("The column nested.flag has a null"),
"{error}"
);
}
}
}
#[test]
fn nested_nulls_are_allowed_under_any_null_ancestor() {
let flag = |nullable| Fields::from(vec![Field::new("flag", DataType::Boolean, nullable)]);
let nulls = |slots: &[bool]| {
let mut builder = NullBufferBuilder::new(slots.len());
builder.append_slice(slots);
builder.finish()
};
// A valid struct with a null flag, under a parent that is null or not.
let structs = || {
let flags: ArrayRef = Arc::new(BooleanArray::from(vec![None::<bool>]));
let structs = StructArray::try_new(flag(true), vec![flags], None).unwrap();
Arc::new(structs) as ArrayRef
};
// struct<inner: struct<flag>>.
let inner = |nullable| Field::new("inner", DataType::Struct(flag(nullable)), true);
let schema = |nullable| {
let nested = DataType::Struct(Fields::from(vec![inner(nullable)]));
Arc::new(Schema::new(vec![Field::new("nested", nested, true)]))
};
for (outer_is_valid, rejected) in [(false, false), (true, true)] {
let fields = Fields::from(vec![inner(true)]);
let outer = StructArray::try_new(fields, vec![structs()], nulls(&[outer_is_valid]));
let batch = RecordBatch::try_new(schema(true), vec![Arc::new(outer.unwrap())]);
let error = batch_error(batch.unwrap(), &schema(false));
assert_eq!(error.is_some(), rejected, "{error:?}");
if let Some(error) = error {
let expected = "The column nested.inner.flag has a null";
assert!(error.contains(expected), "{error}");
}
}
// array<struct<flag>>.
let element = |nullable| Field::new("element", DataType::Struct(flag(nullable)), true);
let schema = |nullable| {
let list = DataType::List(Arc::new(element(nullable)));
Arc::new(Schema::new(vec![Field::new("values", list, true)]))
};
for (list_is_valid, rejected) in [(false, false), (true, true)] {
let mut offsets = OffsetBufferBuilder::<i32>::new(1);
offsets.push_length(1);
let lists = ListArray::try_new(
Arc::new(element(true)),
offsets.finish(),
structs(),
nulls(&[list_is_valid]),
);
let batch = RecordBatch::try_new(schema(true), vec![Arc::new(lists.unwrap())]);
let error = batch_error(batch.unwrap(), &schema(false));
assert_eq!(error.is_some(), rejected, "{error:?}");
if let Some(error) = error {
let expected = "The column values.element.flag has a null";
assert!(error.contains(expected), "{error}");
}
}
}
#[test]
fn list_elements_and_map_values_respect_their_nullability() {
// ListBuilder names the element "item", and Spark names it "element" in the schema to
// read, which the SDK accepts.
let list_schema = |name: &str, nullable| {
let element = Arc::new(Field::new(name, DataType::Int32, nullable));
Arc::new(Schema::new(vec![Field::new(
"values",
DataType::List(element),
true,
)]))
};
for (first, rejected) in [(Some(1), false), (None, true)] {
let mut builder = ListBuilder::new(Int32Builder::new());
builder.values().append_option(first);
builder.append(true);
// The second list is null, so its null element does not count.
builder.values().append_null();
builder.append(false);
let lists = Arc::new(builder.finish());
let batch = RecordBatch::try_new(list_schema("item", true), vec![lists]).unwrap();
let error = batch_error(batch, &list_schema("element", false));
assert_eq!(error.is_some(), rejected, "{error:?}");
if let Some(error) = error {
assert!(
error.contains("The column values.element has a null"),
"{error}"
);
}
}
let mut builder = MapBuilder::new(None, StringBuilder::new(), Int32Builder::new());
builder.keys().append_value("a");
builder.values().append_null();
builder.append(true).unwrap();
let map = builder.finish();
let actual = Arc::new(Schema::new(vec![Field::new(
"map",
map.data_type().clone(),
true,
)]));
let entries = Fields::from(vec![
Field::new("key", DataType::Utf8, false),
Field::new("value", DataType::Int32, false),
]);
let entries = Arc::new(Field::new("entries", DataType::Struct(entries), false));
let required = Arc::new(Schema::new(vec![Field::new(
"map",
DataType::Map(entries, false),
true,
)]));
let batch = RecordBatch::try_new(actual.clone(), vec![Arc::new(map)]).unwrap();
let error = batch_error(batch.clone(), &required);
assert!(error.unwrap().contains("The column map.value has a null"));
assert_eq!(batch_error(batch, &actual), None);
// [{"a": 1}, null], where the null map still spans an entry with a null key and a null
// value: neither counts, even though the schema to read declares both not nullable.
let batch = map_batch(vec![Some(1), None], &[1, 1], &[true, false]);
assert_eq!(batch_error(batch, &required), None);
}
/// Builds a batch with one nullable column "map" over the keys ["a", null] and `values`,
/// with a map of each of `lengths` entries, null where `slots` is false. MapBuilder rejects
/// null keys, but a map whose key field is nullable can have them.
fn map_batch(values: Vec<Option<i32>>, lengths: &[usize], slots: &[bool]) -> RecordBatch {
let fields = Fields::from(vec![
Field::new("key", DataType::Utf8, true),
Field::new("value", DataType::Int32, true),
]);
let keys: ArrayRef = Arc::new(StringArray::from(vec![Some("a"), None]));
let values: ArrayRef = Arc::new(Int32Array::from(values));
let entries = StructArray::try_new(fields.clone(), vec![keys, values], None).unwrap();
let entries_field = Arc::new(Field::new("entries", DataType::Struct(fields), false));
let mut offsets = OffsetBufferBuilder::<i32>::new(lengths.len());
for &length in lengths {
offsets.push_length(length);
}
let mut nulls = NullBufferBuilder::new(slots.len());
nulls.append_slice(slots);
let (offsets, nulls) = (offsets.finish(), nulls.finish());
let map = MapArray::try_new(entries_field, offsets, entries, nulls, false).unwrap();
let schema = Arc::new(Schema::new(vec![Field::new(
"map",
map.data_type().clone(),
true,
)]));
RecordBatch::try_new(schema, vec![Arc::new(map)]).unwrap()
}
#[test]
fn map_keys_cannot_be_null_whatever_the_schema_to_read_declares() {
let batch = map_batch(vec![Some(1), Some(2)], &[2], &[true]);
let error = batch_error(batch.clone(), &batch.schema()).unwrap();
assert!(error.contains("The column map.key has a null"), "{error}");
}
#[test]
fn decimal_values_must_fit_the_precision_of_the_schema_to_read() {
let schema = Arc::new(Schema::new(vec![Field::new(
"amount",
DataType::Decimal128(5, 2),
true,
)]));
let amounts = |values: Vec<Option<i128>>| -> ArrayRef {
let array = Decimal128Array::from(values);
Arc::new(array.with_precision_and_scale(5, 2).unwrap())
};
let fitting = amounts(vec![Some(12345), None, Some(-99999)]);
let batch = RecordBatch::try_new(schema.clone(), vec![fitting]).unwrap();
assert_eq!(batch_error(batch, &schema), None);
let overflowing = amounts(vec![Some(12345), Some(123456)]);
let batch = RecordBatch::try_new(schema.clone(), vec![overflowing]).unwrap();
let error = batch_error(batch, &schema).unwrap();
assert!(
error.contains("The column amount has the value 1234.56"),
"{error}"
);
assert!(error.contains("precision 5"), "{error}");
}
#[test]
fn decimal_values_under_a_null_parent_are_not_checked() {
let fields = Fields::from(vec![Field::new("amount", DataType::Decimal128(5, 2), true)]);
let schema = Arc::new(Schema::new(vec![Field::new(
"nested",
DataType::Struct(fields.clone()),
true,
)]));
for (parent_is_valid, rejected) in [(false, false), (true, true)] {
let amounts = Decimal128Builder::new().with_data_type(DataType::Decimal128(5, 2));
let mut builder = StructBuilder::new(fields.clone(), vec![Box::new(amounts)]);
builder
.field_builder::<Decimal128Builder>(0)
.unwrap()
.append_value(123456);
builder.append(parent_is_valid);
let batch =
RecordBatch::try_new(schema.clone(), vec![Arc::new(builder.finish())]).unwrap();
let error = batch_error(batch, &schema);
assert_eq!(error.is_some(), rejected, "{error:?}");
if let Some(error) = error {
assert!(error.contains("nested.amount"), "{error}");
}
}
}
#[test]
fn a_batch_cannot_change_the_schema_of_the_stream() {
let mut reader = reader();
let first = reader.batches.next().unwrap();
let changed_schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int32, false)]));
let changed =
RecordBatch::try_new(changed_schema, vec![Arc::new(Int32Array::from(vec![7, 9]))])
.unwrap();
reader.batches = vec![first, Ok(changed)].into_iter();
let drops = reader.drops.clone();
let mut stream = ArrowArrayStreamReader::try_new(export(reader)).unwrap();
assert_eq!(stream.next().unwrap().unwrap().num_rows(), 2);
let error = stream.next().unwrap().unwrap_err().to_string();
assert!(error.contains("different schema"), "{error}");
assert_eq!(stream.next().unwrap().unwrap_err().to_string(), error);
drop(stream);
assert_eq!(drops.load(Ordering::SeqCst), 1);
}
#[test]
fn a_batch_cannot_change_the_arrow_encoding_of_a_spark_type() {
// Utf8 and LargeUtf8 are the same Spark type, but every array of the stream is exported
// with the schema of the first batch, so a later batch must have exactly that schema.
let label = |data_type| Arc::new(Schema::new(vec![Field::new("label", data_type, true)]));
let first = RecordBatch::try_new(
label(DataType::Utf8),
vec![Arc::new(StringArray::from(vec!["a"]))],
)
.unwrap();
let second = RecordBatch::try_new(
label(DataType::LargeUtf8),
vec![Arc::new(LargeStringArray::from(vec!["b"]))],
)
.unwrap();
let exported = export_with(reader_of(vec![first, second]), &label(DataType::Utf8));
let mut stream = ArrowArrayStreamReader::try_new(exported).unwrap();
assert_eq!(stream.next().unwrap().unwrap().num_rows(), 1);
let error = stream.next().unwrap().unwrap_err().to_string();
assert!(error.contains("different schema"), "{error}");
assert_eq!(stream.next().unwrap().unwrap_err().to_string(), error);
}
}