| // Licensed to the Apache Software Foundation (ASF) under one or more | |
| // contributor license agreements. See the NOTICE file distributed with | |
| // this work for additional information regarding copyright ownership. | |
| // The ASF licenses this file to You under the Apache License, Version 2.0 | |
| // (the "License"); you may not use this file except in compliance with | |
| // the License. You may obtain a copy of the License at | |
| // | |
| // http://www.apache.org/licenses/LICENSE-2.0 | |
| // | |
| // Unless required by applicable law or agreed to in writing, software | |
| // distributed under the License is distributed on an "AS IS" BASIS, | |
| // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. | |
| // See the License for the specific language governing permissions and | |
| // limitations under the License. | |
| use std::marker::PhantomData; | |
| use std::sync::atomic::{AtomicBool, Ordering}; | |
| use crate::abi::{self, Interrupted, SparkDsTaskContext}; | |
| use crate::Result; | |
| /// Task identifiers and cooperative cancellation, borrowed until its reader or writer is closed. | |
| /// | |
| /// A reader or writer may retain this value and check it during long operations. | |
| /// It cannot retain it beyond the lifetime of its returned reader or writer. | |
| /// Blocking I/O still needs its own timeouts. | |
| /// | |
| /// Partition identifiers belong to [`crate::ReaderContext::partition_index`] and | |
| /// [`crate::WriterContext::partition_id`]: the native ABI supplies a scan partition index | |
| /// for readers, which need not be the Spark task's partition ID. | |
| /// | |
| /// ```compile_fail | |
| /// use spark_datasource::TaskContext; | |
| /// | |
| /// fn retain_forever(task: TaskContext<'_>) -> TaskContext<'static> { | |
| /// task | |
| /// } | |
| /// ``` | |
| #[derive(Debug, Clone, Copy)] | |
| pub struct TaskContext<'task> { | |
| stage_id: i32, | |
| stage_attempt_number: i32, | |
| attempt_number: i32, | |
| task_attempt_id: i64, | |
| interruption: Interruption<'task>, | |
| } | |
| /// Where a task learns that Spark killed it. | |
| #[derive(Debug, Clone, Copy)] | |
| enum Interruption<'task> { | |
| Never, | |
| Host { | |
| context: *const SparkDsTaskContext, | |
| callback: Interrupted, | |
| lifetime: PhantomData<&'task SparkDsTaskContext>, | |
| }, | |
| Flag(&'task AtomicBool), | |
| } | |
| // The C ABI permits task callbacks from any thread. The lifetime prevents use after the | |
| // host releases its context; the remaining fields are copied values or immutable callbacks. | |
| unsafe impl Send for TaskContext<'_> {} | |
| unsafe impl Sync for TaskContext<'_> {} | |
| impl TaskContext<'static> { | |
| /// A context for testing a reader or writer outside Spark. Its IDs are 0 until set | |
| /// with the `with_*` methods, and it is never interrupted unless it is given a flag | |
| /// with [`interrupted_by`](Self::interrupted_by). | |
| pub fn for_testing() -> Self { | |
| Self { | |
| stage_id: 0, | |
| stage_attempt_number: 0, | |
| attempt_number: 0, | |
| task_attempt_id: 0, | |
| interruption: Interruption::Never, | |
| } | |
| } | |
| #[must_use] | |
| pub fn with_stage_id(mut self, stage_id: i32) -> Self { | |
| self.stage_id = stage_id; | |
| self | |
| } | |
| #[must_use] | |
| pub fn with_stage_attempt_number(mut self, stage_attempt_number: i32) -> Self { | |
| self.stage_attempt_number = stage_attempt_number; | |
| self | |
| } | |
| #[must_use] | |
| pub fn with_attempt_number(mut self, attempt_number: i32) -> Self { | |
| self.attempt_number = attempt_number; | |
| self | |
| } | |
| #[must_use] | |
| pub fn with_task_attempt_id(mut self, task_attempt_id: i64) -> Self { | |
| self.task_attempt_id = task_attempt_id; | |
| self | |
| } | |
| /// Reports the task as interrupted whenever `flag` is true. The context borrows the | |
| /// flag, so set its IDs first. | |
| #[must_use] | |
| pub fn interrupted_by(self, flag: &AtomicBool) -> TaskContext<'_> { | |
| TaskContext { | |
| interruption: Interruption::Flag(flag), | |
| ..self | |
| } | |
| } | |
| } | |
| impl TaskContext<'_> { | |
| /// Wraps a host context and returns its operation-specific partition index separately. | |
| /// The context must have every field of version 1 of the ABI. | |
| /// | |
| /// # Safety | |
| /// The host must keep the context and its callbacks valid for `'task`, including calls from | |
| /// other threads. Spark keeps it alive until the Arrow stream is released or | |
| /// data writer closes. | |
| pub(crate) unsafe fn from_raw(pointer: *const SparkDsTaskContext) -> Result<(Self, i32)> { | |
| let context = unsafe { abi::input(pointer)? }; | |
| Ok(( | |
| Self { | |
| stage_id: context.stage_id, | |
| stage_attempt_number: context.stage_attempt_number, | |
| attempt_number: context.attempt_number, | |
| task_attempt_id: context.task_attempt_id, | |
| interruption: match context.is_interrupted { | |
| Some(callback) => Interruption::Host { | |
| context: pointer, | |
| callback, | |
| lifetime: PhantomData, | |
| }, | |
| None => Interruption::Never, | |
| }, | |
| }, | |
| context.partition_id, | |
| )) | |
| } | |
| /// The ID of the stage of the task. | |
| pub fn stage_id(&self) -> i32 { | |
| self.stage_id | |
| } | |
| /// The attempt number of the stage of the task. | |
| pub fn stage_attempt_number(&self) -> i32 { | |
| self.stage_attempt_number | |
| } | |
| /// The attempt number of the task. | |
| pub fn attempt_number(&self) -> i32 { | |
| self.attempt_number | |
| } | |
| /// The ID of the task attempt, unique within the Spark application but not across | |
| /// applications. | |
| pub fn task_attempt_id(&self) -> i64 { | |
| self.task_attempt_id | |
| } | |
| /// Whether Spark killed this task, read from Spark each time. For a reader, also whether | |
| /// the task failed or completed: a native engine may read a stream after its task ended. | |
| pub fn is_interrupted(&self) -> bool { | |
| match self.interruption { | |
| Interruption::Never => false, | |
| Interruption::Host { | |
| context, callback, .. | |
| } => unsafe { callback(context) }, | |
| Interruption::Flag(flag) => flag.load(Ordering::SeqCst), | |
| } | |
| } | |
| } | |
| #[cfg(test)] | |
| mod tests { | |
| use std::mem::size_of; | |
| use std::sync::atomic::{AtomicBool, Ordering}; | |
| use super::*; | |
| #[repr(C)] | |
| struct HostTask { | |
| task: SparkDsTaskContext, | |
| interrupted: AtomicBool, | |
| } | |
| unsafe extern "C" fn interrupted(task: *const SparkDsTaskContext) -> bool { | |
| unsafe { &*task.cast::<HostTask>() } | |
| .interrupted | |
| .load(Ordering::SeqCst) | |
| } | |
| #[test] | |
| fn reads_ids_and_live_cancellation_from_the_host_context() { | |
| let host = HostTask { | |
| task: SparkDsTaskContext { | |
| size: size_of::<SparkDsTaskContext>() as i64, | |
| stage_id: 3, | |
| stage_attempt_number: 4, | |
| partition_id: 5, | |
| attempt_number: 6, | |
| task_attempt_id: 7, | |
| is_interrupted: Some(interrupted), | |
| }, | |
| interrupted: AtomicBool::new(false), | |
| }; | |
| let (context, partition) = unsafe { TaskContext::from_raw(&host.task) }.unwrap(); | |
| assert_eq!(context.stage_id(), 3); | |
| assert_eq!(context.stage_attempt_number(), 4); | |
| assert_eq!(partition, 5); | |
| assert_eq!(context.attempt_number(), 6); | |
| assert_eq!(context.task_attempt_id(), 7); | |
| assert!(!context.is_interrupted()); | |
| host.interrupted.store(true, Ordering::SeqCst); | |
| std::thread::scope(|scope| { | |
| scope.spawn(|| assert!(context.is_interrupted())); | |
| }); | |
| } | |
| #[test] | |
| fn rejects_a_context_without_every_field_of_version_1() { | |
| let prefix = size_of::<i64>() as i64; | |
| let pointer = (&prefix as *const i64).cast::<SparkDsTaskContext>(); | |
| let error = unsafe { TaskContext::from_raw(pointer) }.unwrap_err(); | |
| assert!(error.to_string().contains("version 1"), "{error}"); | |
| assert!(unsafe { TaskContext::from_raw(std::ptr::null()) }.is_err()); | |
| } | |
| #[test] | |
| fn test_contexts_carry_ids_and_an_optional_interruption_flag() { | |
| let context = TaskContext::for_testing() | |
| .with_stage_id(1) | |
| .with_stage_attempt_number(2) | |
| .with_attempt_number(4) | |
| .with_task_attempt_id(5); | |
| assert_eq!( | |
| ( | |
| context.stage_id(), | |
| context.stage_attempt_number(), | |
| context.attempt_number(), | |
| context.task_attempt_id() | |
| ), | |
| (1, 2, 4, 5) | |
| ); | |
| assert!(!context.is_interrupted()); | |
| let flag = AtomicBool::new(false); | |
| let context = TaskContext::for_testing().interrupted_by(&flag); | |
| assert!(!context.is_interrupted()); | |
| flag.store(true, Ordering::SeqCst); | |
| assert!(context.is_interrupted()); | |
| } | |
| } |