| // Licensed to the Apache Software Foundation (ASF) under one or more | |
| // contributor license agreements. See the NOTICE file distributed with | |
| // this work for additional information regarding copyright ownership. | |
| // The ASF licenses this file to You under the Apache License, Version 2.0 | |
| // (the "License"); you may not use this file except in compliance with | |
| // the License. You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| use std::ffi::{c_char, c_int, c_void, CStr, CString}; | |
| use std::mem::size_of; | |
| use std::ptr; | |
| use std::sync::atomic::{AtomicI32, AtomicUsize, Ordering}; | |
| use std::sync::{Arc, Mutex}; | |
| use arrow_schema::{Schema, SchemaRef}; | |
| use serde::{Deserialize, Serialize}; | |
| use spark_datasource::__private::abi::{ | |
| ArrowSchema, SparkDsCreateInfo, SparkDsHost, SparkDsLibrary, SparkDsReaderInfo, ABI_VERSION, | |
| }; | |
| use spark_datasource::log; | |
| use spark_datasource::{ | |
| Capabilities, CreateInfo, DataSource, HostInfo, PartitionReader, PartitionReaderFactory, | |
| ReaderContext, Result, ScanBuilder, ScanInfo, ScanPlan, Unsupported, | |
| }; | |
| static DECLARATIONS: AtomicUsize = AtomicUsize::new(0); | |
| static SCHEMAS: AtomicUsize = AtomicUsize::new(0); | |
| static SCANS: AtomicUsize = AtomicUsize::new(0); | |
| static DROPS: AtomicUsize = AtomicUsize::new(0); | |
| static INITIALIZED: Mutex<Vec<Option<String>>> = Mutex::new(Vec::new()); | |
| static LOGS: Mutex<Vec<(i32, String)>> = Mutex::new(Vec::new()); | |
| // The most verbose level that the log of the host writes, as a SPARK_DS_LOG_* value. | |
| static LEVEL: AtomicI32 = AtomicI32::new(3); | |
| struct Source { | |
| mode: String, | |
| } | |
| impl DataSource for Source { | |
| type ScanBuilder = Builder; | |
| type MicroBatchStream = Unsupported; | |
| type BatchWrite = Unsupported; | |
| type StreamingWrite = Unsupported; | |
| fn init(host: &HostInfo) -> Result<()> { | |
| INITIALIZED.lock().unwrap().push(host.spark_version.clone()); | |
| if host.spark_version.as_deref() == Some("unsupported") { | |
| return Err("this library does not support the Spark version".into()); | |
| } | |
| log::info!("capability test library initialized"); | |
| Ok(()) | |
| } | |
| fn new(info: CreateInfo) -> Result<Self> { | |
| if info.name != "capability_test" { | |
| return Err(format!("unexpected source name {}", info.name).into()); | |
| } | |
| Ok(Self { | |
| mode: info.options.get("capabilities").cloned().unwrap(), | |
| }) | |
| } | |
| fn capabilities(&self) -> Capabilities { | |
| DECLARATIONS.fetch_add(1, Ordering::SeqCst); | |
| match self.mode.as_str() { | |
| "empty" => Capabilities::empty(), | |
| "read" => Capabilities::BATCH_READ, | |
| "without_read" => Capabilities::all().without(Capabilities::BATCH_READ), | |
| // Operations that the source does not implement are dropped. | |
| "all" => Capabilities::all(), | |
| "panic" => panic!("capability discovery failed"), | |
| other => panic!("unexpected test mode: {other}"), | |
| } | |
| } | |
| fn infer_schema(&self) -> Result<SchemaRef> { | |
| SCHEMAS.fetch_add(1, Ordering::SeqCst); | |
| Ok(Arc::new(Schema::empty())) | |
| } | |
| fn new_scan(&self, _: ScanInfo) -> Result<Builder> { | |
| SCANS.fetch_add(1, Ordering::SeqCst); | |
| Ok(Builder) | |
| } | |
| } | |
| impl Drop for Source { | |
| fn drop(&mut self) { | |
| DROPS.fetch_add(1, Ordering::SeqCst); | |
| } | |
| } | |
| struct Builder; | |
| impl ScanBuilder for Builder { | |
| type Factory = Factory; | |
| fn plan(self) -> Result<ScanPlan<Factory>> { | |
| Ok(ScanPlan::new(Factory, vec![])) | |
| } | |
| } | |
| #[derive(Serialize, Deserialize)] | |
| struct Factory; | |
| impl PartitionReaderFactory for Factory { | |
| type Partition = (); | |
| fn open<'task>(&self, _: (), _: ReaderContext<'task>) -> Result<PartitionReader<'task>> { | |
| Err("capability tests do not open partition readers".into()) | |
| } | |
| } | |
| spark_datasource::export_data_source!(Source); | |
| extern "C" { | |
| fn spark_ds_init(host: *const SparkDsHost, library: *mut *const SparkDsLibrary) -> c_int; | |
| } | |
| unsafe extern "C" fn record_log(level: i32, message: *const c_char) { | |
| let message = unsafe { CStr::from_ptr(message) } | |
| .to_str() | |
| .unwrap() | |
| .to_owned(); | |
| LOGS.lock().unwrap().push((level, message)); | |
| } | |
| unsafe extern "C" fn is_log_enabled(level: i32) -> bool { | |
| level <= LEVEL.load(Ordering::SeqCst) | |
| } | |
| /// A host that, like Spark's, stays valid until the process exits. | |
| fn host(spark_version: &'static CStr) -> &'static SparkDsHost { | |
| Box::leak(Box::new(SparkDsHost { | |
| size: size_of::<SparkDsHost>() as i64, | |
| abi_version: ABI_VERSION, | |
| spark_version: spark_version.as_ptr(), | |
| log: Some(record_log), | |
| is_log_enabled: Some(is_log_enabled), | |
| })) | |
| } | |
| struct SourceHandle<'a> { | |
| library: &'a SparkDsLibrary, | |
| pointer: *mut c_void, | |
| } | |
| impl Drop for SourceHandle<'_> { | |
| fn drop(&mut self) { | |
| unsafe { self.library.close.unwrap()(self.pointer) }; | |
| } | |
| } | |
| fn create(library: &SparkDsLibrary, name: &str, mode: &str) -> (c_int, u64, *mut c_void) { | |
| let name = CString::new(name).unwrap(); | |
| let key = CString::new("capabilities").unwrap(); | |
| let value = CString::new(mode).unwrap(); | |
| let keys = [key.as_ptr()]; | |
| let values = [value.as_ptr()]; | |
| let info = SparkDsCreateInfo { | |
| size: size_of::<SparkDsCreateInfo>() as i64, | |
| name: name.as_ptr(), | |
| option_keys: keys.as_ptr(), | |
| option_values: values.as_ptr(), | |
| num_options: 1, | |
| }; | |
| let mut mask = 0; | |
| let mut source = ptr::null_mut(); | |
| let status = unsafe { library.create.unwrap()(&info, &mut mask, &mut source) }; | |
| (status, mask, source) | |
| } | |
| fn error(library: &SparkDsLibrary) -> String { | |
| let message = unsafe { library.last_error.unwrap()() }; | |
| assert!(!message.is_null(), "failed C call must set last_error"); | |
| unsafe { CStr::from_ptr(message) }.to_str().unwrap().into() | |
| } | |
| #[test] | |
| fn initialization_logging_and_capabilities_follow_the_source_lifecycle() { | |
| // One lifecycle test owns these counters; no parallel test resets shared state. | |
| let mut table = ptr::null(); | |
| assert_ne!( | |
| unsafe { spark_ds_init(host(c"unsupported"), &mut table) }, | |
| 0 | |
| ); | |
| assert!( | |
| table.is_null(), | |
| "a library whose init fails returns no table" | |
| ); | |
| assert_eq!( | |
| LOGS.lock().unwrap().as_slice(), | |
| [( | |
| 1, | |
| "this library does not support the Spark version".to_owned() | |
| )], | |
| ); | |
| assert_eq!(unsafe { spark_ds_init(host(c"4.4.0-test"), &mut table) }, 0); | |
| assert!(!table.is_null()); | |
| let library = unsafe { &*table }; | |
| assert_eq!( | |
| INITIALIZED.lock().unwrap().as_slice(), | |
| [ | |
| Some("unsupported".to_owned()), | |
| Some("4.4.0-test".to_owned()) | |
| ], | |
| ); | |
| log::warn!("a message with a \0 byte"); | |
| // The SDK follows the level of the log of Spark, which drops debug messages. | |
| log::debug!("a dropped debug message"); | |
| // The level of Spark changes while it runs: the SDK asks again when an operation starts. | |
| LEVEL.store(4, Ordering::SeqCst); | |
| log::debug!("a debug message before an operation starts"); | |
| let (status, _, pointer) = create(library, "capability_test", "all"); | |
| assert_eq!(status, 0); | |
| log::debug!("a debug message"); | |
| log::trace!("a dropped trace message"); | |
| drop(SourceHandle { library, pointer }); | |
| LEVEL.store(3, Ordering::SeqCst); | |
| assert_eq!( | |
| LOGS.lock().unwrap()[1..], | |
| [ | |
| (3, "capability test library initialized".to_owned()), | |
| (2, "a message with a \\0 byte".to_owned()), | |
| (4, "a debug message".to_owned()), | |
| ], | |
| ); | |
| // Analysis can ask only for capabilities, then close without schema or scan discovery. | |
| let (status, mask, pointer) = create(library, "capability_test", "all"); | |
| assert_eq!(status, 0); | |
| assert_eq!(mask, 1); | |
| assert!(!pointer.is_null()); | |
| drop(SourceHandle { library, pointer }); | |
| assert_eq!(DECLARATIONS.load(Ordering::SeqCst), 2); | |
| assert_eq!(SCHEMAS.load(Ordering::SeqCst), 0); | |
| assert_eq!(SCANS.load(Ordering::SeqCst), 0); | |
| assert_eq!(DROPS.load(Ordering::SeqCst), 2); | |
| // Keep all instances alive together: the last declaration must not affect earlier sources. | |
| let sources: Vec<_> = [("empty", 0), ("read", 1), ("all", 1), ("without_read", 0)] | |
| .into_iter() | |
| .map(|(mode, expected)| { | |
| let (status, mask, pointer) = create(library, "capability_test", mode); | |
| assert_eq!(status, 0); | |
| assert_eq!(mask, expected, "{mode}"); | |
| assert!(!pointer.is_null()); | |
| (expected, SourceHandle { library, pointer }) | |
| }) | |
| .collect(); | |
| assert_eq!(DECLARATIONS.load(Ordering::SeqCst), 6); | |
| let info = SparkDsReaderInfo { | |
| size: size_of::<SparkDsReaderInfo>() as i64, | |
| }; | |
| for (mask, source) in sources { | |
| let mut schema = ArrowSchema::empty(); | |
| assert_eq!( | |
| unsafe { library.schema.unwrap()(source.pointer, &mut schema) }, | |
| 0 | |
| ); | |
| let scans_before = SCANS.load(Ordering::SeqCst); | |
| let mut reader = ptr::null_mut(); | |
| let status = unsafe { | |
| library.create_reader.unwrap()(source.pointer, &mut schema, &info, &mut reader) | |
| }; | |
| if mask == 0 { | |
| assert_ne!(status, 0); | |
| assert_eq!(error(library), "source does not support batch reads"); | |
| assert!(reader.is_null()); | |
| assert_eq!(SCANS.load(Ordering::SeqCst), scans_before); | |
| } else { | |
| assert_eq!(status, 0); | |
| assert!(!reader.is_null()); | |
| assert_eq!(SCANS.load(Ordering::SeqCst), scans_before + 1); | |
| unsafe { library.close_reader.unwrap()(reader) }; | |
| } | |
| assert_eq!(DECLARATIONS.load(Ordering::SeqCst), 6); | |
| } | |
| assert_eq!(SCHEMAS.load(Ordering::SeqCst), 4); | |
| assert_eq!(SCANS.load(Ordering::SeqCst), 2); | |
| assert_eq!(DROPS.load(Ordering::SeqCst), 6); | |
| // The source receives the name that Spark passes, so it can reject other names. | |
| let (status, mask, source) = create(library, "other_source", "all"); | |
| assert_ne!(status, 0); | |
| assert_eq!(error(library), "unexpected source name other_source"); | |
| assert_eq!(mask, 0); | |
| assert!(source.is_null()); | |
| let drops_before = DROPS.load(Ordering::SeqCst); | |
| let (status, mask, source) = create(library, "capability_test", "panic"); | |
| assert_ne!(status, 0); | |
| assert_eq!( | |
| error(library), | |
| "native provider panicked: capability discovery failed" | |
| ); | |
| assert_eq!(mask, 0); | |
| assert!(source.is_null()); | |
| assert_eq!(DROPS.load(Ordering::SeqCst), drops_before + 1); | |
| assert_eq!(SCHEMAS.load(Ordering::SeqCst), 4); | |
| assert_eq!(SCANS.load(Ordering::SeqCst), 2); | |
| } |