feat(ruby): prototype shared runtime adapter
diff --git a/bindings/python/dynamic-extensions/audit-elf-exports.sh b/bindings/python/dynamic-extensions/audit-elf-exports.sh index 59a84bb..97845f6 100755 --- a/bindings/python/dynamic-extensions/audit-elf-exports.sh +++ b/bindings/python/dynamic-extensions/audit-elf-exports.sh
@@ -16,13 +16,14 @@ set -eu -if [ "$#" -ne 2 ]; then - echo "usage: $0 ARTIFACT EXPECTED_SYMBOL" >&2 +if [ "$#" -lt 2 ]; then + echo "usage: $0 ARTIFACT EXPECTED_SYMBOL..." >&2 exit 2 fi artifact=$1 -expected=$2 +shift +expected=$(printf '%s\n' "$@" | LC_ALL=C sort -u) actual=$( nm --dynamic --defined-only --extern-only --format=posix "$artifact" \ | awk '{ print $1 }' \ @@ -38,4 +39,4 @@ exit 1 fi -echo "$artifact exports only $expected" +echo "$artifact exports only the expected symbols"
diff --git a/bindings/ruby/dynamic-extensions/.gitignore b/bindings/ruby/dynamic-extensions/.gitignore new file mode 100644 index 0000000..d7d2adf --- /dev/null +++ b/bindings/ruby/dynamic-extensions/.gitignore
@@ -0,0 +1,16 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +/target/
diff --git a/bindings/ruby/dynamic-extensions/Cargo.lock b/bindings/ruby/dynamic-extensions/Cargo.lock new file mode 100644 index 0000000..f6b68fb --- /dev/null +++ b/bindings/ruby/dynamic-extensions/Cargo.lock
@@ -0,0 +1,297 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags", + "cexpr", + "clang-sys", + "itertools", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bitflags" +version = "2.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" + +[[package]] +name = "bytes" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04" + +[[package]] +name = "cexpr" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766" +dependencies = [ + "nom", +] + +[[package]] +name = "cfg-if" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" + +[[package]] +name = "clang-sys" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" +dependencies = [ + "glob", + "libc", + "libloading", +] + +[[package]] +name = "either" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" + +[[package]] +name = "glob" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b" + +[[package]] +name = "itertools" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "413ee7dfc52ee1a4949ceeb7dbc8a33f2d6c088194d9f922fb8318faf1f01186" +dependencies = [ + "either", +] + +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + +[[package]] +name = "libc" +version = "0.2.189" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" + +[[package]] +name = "libloading" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" +dependencies = [ + "cfg-if", + "windows-link", +] + +[[package]] +name = "magnus" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b36a5b126bbe97eb0d02d07acfeb327036c6319fd816139a49824a83b7f9012" +dependencies = [ + "bytes", + "magnus-macros", + "rb-sys", + "rb-sys-env", + "seq-macro", +] + +[[package]] +name = "magnus-macros" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47607461fd8e1513cb4f2076c197d8092d921a1ea75bd08af97398f593751892" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "memchr" +version = "2.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "opendal-dynamic-extension-sdk" +version = "0.0.0" + +[[package]] +name = "opendal-ruby-runtime-poc" +version = "0.0.0" +dependencies = [ + "bytes", + "libloading", + "magnus", + "opendal-dynamic-extension-sdk", + "rb-sys", + "rb-sys-env", +] + +[[package]] +name = "proc-macro2" +version = "1.0.107" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "985e7ec9bb745e6ce6535b544d84d6cd6f7ad8bd711c398938ae983b91a766d9" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.47" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fbf4db142a473a8d80c26bbf18454ed458bf8d26c8219c331daecfdbd079001" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rb-sys" +version = "0.9.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "45ca28513560e56cfb79a62b1fce363c73af170a182024ce880c77ee9429920a" +dependencies = [ + "rb-sys-build", +] + +[[package]] +name = "rb-sys-build" +version = "0.9.128" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce04b2c55eff3a21aaa623fcc655d94373238e72cac6b3e1a3641ff31649f99a" +dependencies = [ + "bindgen", + "lazy_static", + "proc-macro2", + "quote", + "regex", + "shell-words", + "syn", +] + +[[package]] +name = "rb-sys-env" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cca7ad6a7e21e72151d56fe2495a259b5670e204c3adac41ee7ef676ea08117a" + +[[package]] +name = "regex" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f020237b6c8eed93db2e2cb53c00c60a8e1bc73da7d073199a1180401450218d" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fcfdb36bda0c880c5931cdc7a2bcdc8ba4556847b9d912bca70bc94708711ad" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4" + +[[package]] +name = "rustc-hash" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" + +[[package]] +name = "seq-macro" +version = "0.3.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" + +[[package]] +name = "shell-words" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc6fe69c597f9c37bfeeeeeb33da3530379845f10be461a66d16d03eca2ded77" + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "syn" +version = "2.0.119" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "872831b642d1a07999a962a351ed35b955ea2cfc8f3862091e2a240a84f17297" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "unicode-ident" +version = "1.0.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" + +[[package]] +name = "windows-link" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
diff --git a/bindings/ruby/dynamic-extensions/Cargo.toml b/bindings/ruby/dynamic-extensions/Cargo.toml new file mode 100644 index 0000000..14107be --- /dev/null +++ b/bindings/ruby/dynamic-extensions/Cargo.toml
@@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[workspace] +members = ["adapter"] +resolver = "2" + +[workspace.package] +edition = "2024" +license = "Apache-2.0" +publish = false +rust-version = "1.91" +version = "0.0.0"
diff --git a/bindings/ruby/dynamic-extensions/NOTES.md b/bindings/ruby/dynamic-extensions/NOTES.md new file mode 100644 index 0000000..7297902 --- /dev/null +++ b/bindings/ruby/dynamic-extensions/NOTES.md
@@ -0,0 +1,50 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# Prototype Notes + +Question: Can Ruby use the same shared runtime and native extension artifacts +as Python without linking a binding-private OpenDAL or Tokio graph? + +Observed result: + +- The Ruby adapter negotiates runtime protocol 1 and rejects a binding that + requires protocol 2. +- Registering the FS Ruby package does not load its native library. Constructing + the first FS operator activates it. +- Ruby writes and reads an 8 KiB payload through the runtime-owned handle. The + adapter exercises the output-buffer resize contract and rejects use after + `Operator#close`. +- The Ruby adapter links Magnus, the protocol SDK, and the native loader. It + does not link `opendal-core`, a service crate, or Tokio. +- The same physical runtime and FS artifacts complete a second FS round trip + through the Python adapter. +- The runtime, FS extension, and Ruby adapter match their explicit ELF export + allowlists. Ruby requires both `Init_opendal_ruby_poc` and + `ruby_abi_version`. + +Production gate: the current runtime protocol owns service registration and an +operation callback table, but it has no layer registration, `LayerHandle`, or +operator-layer composition function. A successful FS run proves cross-language +runtime reuse only. It does not satisfy the design requirement that arbitrary +native layers preserve `apply_service` and `apply_context` semantics. + +Decision: do not migrate the production Python and Ruby bindings yet. The +current runtime implements the Design B operation-table boundary for services, +not the selected Design C shared OpenDAL graph. Production work first needs a +runtime-owned service/layer factory interface and a real operation that proves +an independently packaged native layer composes through that graph.
diff --git a/bindings/ruby/dynamic-extensions/README.md b/bindings/ruby/dynamic-extensions/README.md new file mode 100644 index 0000000..338a587 --- /dev/null +++ b/bindings/ruby/dynamic-extensions/README.md
@@ -0,0 +1,40 @@ +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to you under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +# Ruby Shared Runtime Prototype + +PROTOTYPE: delete or absorb this directory after the design question is +answered. + +Question: Can a Ruby binding adapter use the exact language-neutral runtime and +FS service artifacts used by the Python dynamic-extension POC? + +The Magnus adapter depends on the extension SDK and loader only. It does not +link OpenDAL core, an OpenDAL service, or Tokio. The runner builds and stages +the existing `opendal-runtime-poc` and `fs-extension` artifacts without +recompiling them for Ruby. + +Run the experiment on Linux: + +```console +./run-ruby-linux.sh +``` + +The runner verifies protocol negotiation, lazy FS activation, real FS +write/read, runtime-owned handle destruction, rejection of a newer binding +protocol, and final ELF export allowlists. It also stages the same runtime and +FS binaries under the Python adapter and completes a second real FS round trip.
diff --git a/bindings/ruby/dynamic-extensions/adapter/Cargo.toml b/bindings/ruby/dynamic-extensions/adapter/Cargo.toml new file mode 100644 index 0000000..618c185 --- /dev/null +++ b/bindings/ruby/dynamic-extensions/adapter/Cargo.toml
@@ -0,0 +1,36 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[package] +name = "opendal-ruby-runtime-poc" +edition.workspace = true +license.workspace = true +publish.workspace = true +rust-version.workspace = true +version.workspace = true + +[lib] +crate-type = ["cdylib"] +name = "opendal_ruby_poc" + +[dependencies] +bytes = "1" +libloading = "0.8.9" +magnus = { version = "0.8", features = ["bytes"] } +opendal-dynamic-extension-sdk = { path = "../../../python/dynamic-extensions/sdk" } +rb-sys = { version = "0.9.110", default-features = false } + +[build-dependencies] +rb-sys-env = "0.2.3"
diff --git a/bindings/ruby/dynamic-extensions/adapter/build.rs b/bindings/ruby/dynamic-extensions/adapter/build.rs new file mode 100644 index 0000000..d41d49b --- /dev/null +++ b/bindings/ruby/dynamic-extensions/adapter/build.rs
@@ -0,0 +1,31 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to you under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::env; +use std::path::PathBuf; + +fn main() -> Result<(), Box<dyn std::error::Error>> { + let _ = rb_sys_env::activate()?; + if env::var("CARGO_CFG_TARGET_OS").as_deref() == Ok("linux") { + let manifest_dir = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); + let export_map = manifest_dir.join("exports.map"); + println!("cargo:rerun-if-changed={}", export_map.display()); + println!( + "cargo:rustc-cdylib-link-arg=-Wl,--version-script={}", + export_map.display() + ); + } + Ok(()) +}
diff --git a/bindings/ruby/dynamic-extensions/adapter/exports.map b/bindings/ruby/dynamic-extensions/adapter/exports.map new file mode 100644 index 0000000..f1e9b63 --- /dev/null +++ b/bindings/ruby/dynamic-extensions/adapter/exports.map
@@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +OPENDAL_RUBY_POC_1 { + global: + Init_opendal_ruby_poc; + local: + *; +};
diff --git a/bindings/ruby/dynamic-extensions/adapter/src/lib.rs b/bindings/ruby/dynamic-extensions/adapter/src/lib.rs new file mode 100644 index 0000000..ac83522 --- /dev/null +++ b/bindings/ruby/dynamic-extensions/adapter/src/lib.rs
@@ -0,0 +1,329 @@ +// Licensed to the Apache Software Foundation (ASF) under one or more +// contributor license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright ownership. +// The ASF licenses this file to you under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance with +// the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +use std::collections::HashMap; +use std::ffi::c_void; +use std::mem::size_of; +use std::path::{Path, PathBuf}; +use std::ptr; +use std::sync::OnceLock; +use std::sync::atomic::{AtomicUsize, Ordering}; + +use libloading::{Library, Symbol}; +use magnus::prelude::*; +use magnus::{Error, RString, Ruby, function, method}; +use opendal_dynamic_extension_sdk::{ + ByteSlice, KeyValue, OutputBuffer, RuntimeApiV1, RuntimeProtocolInfoV1, + STATUS_BUFFER_TOO_SMALL, STATUS_OK, ServiceRegistrationV1, +}; + +type GetRuntimeApiFn = + unsafe extern "C" fn(u32, *mut RuntimeProtocolInfoV1, *mut *const RuntimeApiV1) -> i32; + +struct NativeRuntime { + _library: Library, + api_address: usize, + library_path: PathBuf, + minimum_protocol: u32, + protocol: u32, +} + +impl NativeRuntime { + fn api(&self) -> &RuntimeApiV1 { + unsafe { &*(self.api_address as *const RuntimeApiV1) } + } +} + +static RUNTIME: OnceLock<NativeRuntime> = OnceLock::new(); + +fn runtime_error(ruby: &Ruby, message: impl AsRef<str>) -> Error { + Error::new(ruby.exception_runtime_error(), message.as_ref().to_owned()) +} + +fn output(capacity: usize) -> (OutputBuffer, Vec<u8>) { + let mut storage = vec![0; capacity]; + let buffer = OutputBuffer { + data: storage.as_mut_ptr(), + capacity: storage.len(), + len: 0, + }; + (buffer, storage) +} + +fn output_message(buffer: &OutputBuffer, storage: &[u8]) -> String { + String::from_utf8_lossy(&storage[..buffer.len.min(storage.len())]).into_owned() +} + +fn byte_slice(value: &[u8]) -> ByteSlice { + ByteSlice { + data: value.as_ptr(), + len: value.len(), + } +} + +fn require_runtime(ruby: &Ruby) -> Result<&'static NativeRuntime, Error> { + RUNTIME + .get() + .ok_or_else(|| runtime_error(ruby, "OpenDAL runtime is not loaded")) +} + +fn load_runtime(ruby: &Ruby, library_path: String, required_protocol: u32) -> Result<(), Error> { + let library_path = Path::new(&library_path) + .canonicalize() + .map_err(|error| runtime_error(ruby, error.to_string()))?; + if let Some(runtime) = RUNTIME.get() { + if runtime.library_path != library_path { + return Err(runtime_error( + ruby, + format!( + "runtime already loaded from {}", + runtime.library_path.display() + ), + )); + } + if required_protocol < runtime.minimum_protocol || required_protocol > runtime.protocol { + return Err(runtime_error(ruby, "runtime protocol is incompatible")); + } + return Ok(()); + } + + let library = unsafe { Library::new(&library_path) } + .map_err(|error| runtime_error(ruby, error.to_string()))?; + let get_api: Symbol<'_, GetRuntimeApiFn> = unsafe { + library + .get(b"opendal_runtime_get_api_v1") + .map_err(|error| runtime_error(ruby, error.to_string()))? + }; + let mut protocol = RuntimeProtocolInfoV1 { + struct_size: size_of::<RuntimeProtocolInfoV1>(), + minimum_runtime_protocol: 0, + runtime_protocol: 0, + }; + let mut api = ptr::null(); + let status = unsafe { get_api(required_protocol, &mut protocol, &mut api) }; + if status != STATUS_OK || api.is_null() { + return Err(runtime_error( + ruby, + format!( + "runtime protocol negotiation failed: required={required_protocol}, supported={}..{}", + protocol.minimum_runtime_protocol, protocol.runtime_protocol + ), + )); + } + + RUNTIME + .set(NativeRuntime { + _library: library, + api_address: api as usize, + library_path, + minimum_protocol: protocol.minimum_runtime_protocol, + protocol: protocol.runtime_protocol, + }) + .map_err(|_| runtime_error(ruby, "runtime was loaded concurrently")) +} + +fn minimum_runtime_protocol(ruby: &Ruby) -> Result<u32, Error> { + Ok(require_runtime(ruby)?.minimum_protocol) +} + +fn runtime_protocol(ruby: &Ruby) -> Result<u32, Error> { + Ok(require_runtime(ruby)?.protocol) +} + +fn register_service( + ruby: &Ruby, + package_id: String, + component_id: String, + entry_symbol: String, + library_path: String, + required_protocol: u32, +) -> Result<(), Error> { + let runtime = require_runtime(ruby)?; + let package = byte_slice(package_id.as_bytes()); + let component = byte_slice(component_id.as_bytes()); + let entry = byte_slice(entry_symbol.as_bytes()); + let path = Path::new(&library_path) + .canonicalize() + .map_err(|error| runtime_error(ruby, error.to_string()))?; + let path = path.to_string_lossy(); + let path_slice = byte_slice(path.as_bytes()); + let registration = ServiceRegistrationV1 { + struct_size: size_of::<ServiceRegistrationV1>(), + required_runtime_protocol: required_protocol, + package_id: package, + component_id: component, + entry_symbol: entry, + library_path: path_slice, + }; + let (mut error, error_storage) = output(4096); + let status = unsafe { (runtime.api().register_service)(®istration, &mut error) }; + if status != STATUS_OK { + return Err(runtime_error(ruby, output_message(&error, &error_storage))); + } + Ok(()) +} + +#[magnus::wrap(class = "OpenDal::Operator", free_immediately)] +struct Operator { + handle_address: AtomicUsize, +} + +impl Drop for Operator { + fn drop(&mut self) { + self.close(); + } +} + +impl Operator { + fn close(&self) { + let handle_address = self.handle_address.swap(0, Ordering::AcqRel); + if handle_address == 0 { + return; + } + if let Some(runtime) = RUNTIME.get() { + unsafe { + (runtime.api().operator_destroy)(handle_address as *mut c_void); + } + } + } + + fn require_handle(&self, ruby: &Ruby) -> Result<*mut c_void, Error> { + let handle_address = self.handle_address.load(Ordering::Acquire); + if handle_address == 0 { + return Err(runtime_error(ruby, "operator is closed")); + } + Ok(handle_address as *mut c_void) + } + + fn new( + ruby: &Ruby, + scheme: String, + options: Option<HashMap<String, String>>, + ) -> Result<Self, Error> { + let runtime = require_runtime(ruby)?; + let scheme = scheme.trim().to_lowercase().replace('_', "-"); + let options = options.unwrap_or_default(); + let pairs: Vec<KeyValue> = options + .iter() + .map(|(key, value)| KeyValue { + key: byte_slice(key.as_bytes()), + value: byte_slice(value.as_bytes()), + }) + .collect(); + let mut handle = ptr::null_mut(); + let (mut error, error_storage) = output(4096); + let status = unsafe { + (runtime.api().create_operator)( + byte_slice(scheme.as_bytes()), + pairs.as_ptr(), + pairs.len(), + &mut handle, + &mut error, + ) + }; + if status != STATUS_OK || handle.is_null() { + return Err(runtime_error(ruby, output_message(&error, &error_storage))); + } + Ok(Self { + handle_address: AtomicUsize::new(handle as usize), + }) + } + + fn close_ruby(operator: &Self) { + operator.close(); + } + + fn info_json(ruby: &Ruby, operator: &Self) -> Result<String, Error> { + let runtime = require_runtime(ruby)?; + let (mut result, result_storage) = output(4096); + let (mut error, error_storage) = output(4096); + let status = unsafe { + (runtime.api().operator_info)(operator.require_handle(ruby)?, &mut result, &mut error) + }; + if status != STATUS_OK { + return Err(runtime_error(ruby, output_message(&error, &error_storage))); + } + Ok(output_message(&result, &result_storage)) + } + + fn write(ruby: &Ruby, operator: &Self, path: String, data: RString) -> Result<(), Error> { + let runtime = require_runtime(ruby)?; + let (mut error, error_storage) = output(4096); + let data = data.to_bytes(); + let status = unsafe { + (runtime.api().operator_write)( + operator.require_handle(ruby)?, + byte_slice(path.as_bytes()), + byte_slice(&data), + &mut error, + ) + }; + if status != STATUS_OK { + return Err(runtime_error(ruby, output_message(&error, &error_storage))); + } + Ok(()) + } + + fn read(ruby: &Ruby, operator: &Self, path: String) -> Result<bytes::Bytes, Error> { + let runtime = require_runtime(ruby)?; + let (mut result, mut result_storage) = output(4096); + let (mut error, error_storage) = output(4096); + let mut status = unsafe { + (runtime.api().operator_read)( + operator.require_handle(ruby)?, + byte_slice(path.as_bytes()), + &mut result, + &mut error, + ) + }; + if status == STATUS_BUFFER_TOO_SMALL { + (result, result_storage) = output(result.len); + status = unsafe { + (runtime.api().operator_read)( + operator.require_handle(ruby)?, + byte_slice(path.as_bytes()), + &mut result, + &mut error, + ) + }; + } + if status != STATUS_OK { + return Err(runtime_error(ruby, output_message(&error, &error_storage))); + } + result_storage.truncate(result.len); + Ok(result_storage.into()) + } +} + +#[magnus::init(name = "opendal_ruby_poc")] +fn init(ruby: &Ruby) -> Result<(), Error> { + let opendal = ruby.define_module("OpenDal")?; + let runtime = opendal.define_module("Runtime")?; + runtime.define_singleton_method("load", function!(load_runtime, 2))?; + runtime.define_singleton_method( + "minimum_runtime_protocol", + function!(minimum_runtime_protocol, 0), + )?; + runtime.define_singleton_method("runtime_protocol", function!(runtime_protocol, 0))?; + runtime.define_singleton_method("register_service", function!(register_service, 5))?; + + let operator = opendal.define_class("Operator", ruby.class_object())?; + operator.define_singleton_method("new", function!(Operator::new, 2))?; + operator.define_method("close", method!(Operator::close_ruby, 0))?; + operator.define_method("info_json", method!(Operator::info_json, 0))?; + operator.define_method("write", method!(Operator::write, 2))?; + operator.define_method("read", method!(Operator::read, 1))?; + Ok(()) +}
diff --git a/bindings/ruby/dynamic-extensions/ruby/example.rb b/bindings/ruby/dynamic-extensions/ruby/example.rb new file mode 100644 index 0000000..5379ca2 --- /dev/null +++ b/bindings/ruby/dynamic-extensions/ruby/example.rb
@@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# frozen_string_literal: true + +require "tmpdir" +require "opendal" +require "opendal/services/fs" + +Dir.mktmpdir("opendal-ruby-runtime-poc-") do |root| + operator = OpenDal::Operator.new("fs", {"root" => root}) + operator.write("hello.txt", "Hello from Ruby through the shared runtime!") + puts operator.info + puts operator.read("hello.txt") + operator.close +end
diff --git a/bindings/ruby/dynamic-extensions/ruby/main/lib/opendal.rb b/bindings/ruby/dynamic-extensions/ruby/main/lib/opendal.rb new file mode 100644 index 0000000..c0b9f77 --- /dev/null +++ b/bindings/ruby/dynamic-extensions/ruby/main/lib/opendal.rb
@@ -0,0 +1,37 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# frozen_string_literal: true + +require "json" +require_relative "opendal_ruby_poc" + +module OpenDal + REQUIRED_RUNTIME_PROTOCOL = Integer( + ENV.fetch("OPENDAL_POC_REQUIRED_RUNTIME_PROTOCOL", "1"), + 10 + ) + + Runtime.load( + File.expand_path("opendal/_native/libopendal_runtime_poc.so", __dir__), + REQUIRED_RUNTIME_PROTOCOL + ) + + class Operator + def info + JSON.parse(info_json) + end + end +end
diff --git a/bindings/ruby/dynamic-extensions/ruby/python_cross_check.py b/bindings/ruby/dynamic-extensions/ruby/python_cross_check.py new file mode 100644 index 0000000..24238c8 --- /dev/null +++ b/bindings/ruby/dynamic-extensions/ruby/python_cross_check.py
@@ -0,0 +1,26 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from tempfile import TemporaryDirectory + +import opendal.services.fs +from opendal import Operator + +with TemporaryDirectory(prefix="opendal-shared-artifact-poc-") as root: + with Operator("fs", root=root) as operator: + operator.write("python.txt", b"same runtime and FS artifacts") + assert operator.read("python.txt") == b"same runtime and FS artifacts" + +print({"adapter": "ctypes", "same_native_artifacts": True, "fs_round_trip": True})
diff --git a/bindings/ruby/dynamic-extensions/ruby/service-fs/lib/opendal/services/fs.rb b/bindings/ruby/dynamic-extensions/ruby/service-fs/lib/opendal/services/fs.rb new file mode 100644 index 0000000..f8b6d64 --- /dev/null +++ b/bindings/ruby/dynamic-extensions/ruby/service-fs/lib/opendal/services/fs.rb
@@ -0,0 +1,39 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# frozen_string_literal: true + +require "opendal" + +module OpenDal + module Services + module Fs + MANIFEST = { + required_runtime_protocol: 1, + package_id: "opendal-service-fs-poc", + component_id: "fs", + native_entry_symbol: "opendal_service_fs_bootstrap_v1" + }.freeze + + Runtime.register_service( + MANIFEST.fetch(:package_id), + MANIFEST.fetch(:component_id), + MANIFEST.fetch(:native_entry_symbol), + File.expand_path("fs/_native/libfs_extension.so", __dir__), + MANIFEST.fetch(:required_runtime_protocol) + ) + end + end +end
diff --git a/bindings/ruby/dynamic-extensions/ruby/test_poc.rb b/bindings/ruby/dynamic-extensions/ruby/test_poc.rb new file mode 100644 index 0000000..b348442 --- /dev/null +++ b/bindings/ruby/dynamic-extensions/ruby/test_poc.rb
@@ -0,0 +1,82 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# frozen_string_literal: true + +require "open3" +require "rbconfig" +require "tmpdir" +require "opendal" + +def assert(condition, message) + raise message unless condition +end + +def loaded?(basename) + File.read("/proc/self/maps").include?(basename) +end + +assert(OpenDal::Runtime.minimum_runtime_protocol == 1, "unexpected minimum protocol") +assert(OpenDal::Runtime.runtime_protocol == 1, "unexpected current protocol") +assert(!loaded?("libfs_extension.so"), "FS loaded before package registration") + +begin + OpenDal::Operator.new("fs", {}) + raise "unregistered FS construction succeeded" +rescue RuntimeError => error + assert(error.message.include?("not registered"), "unexpected registration error") +end + +require "opendal/services/fs" +assert(!loaded?("libfs_extension.so"), "FS loaded during package registration") + +Dir.mktmpdir("opendal-ruby-runtime-poc-") do |root| + operator = OpenDal::Operator.new("fs", {"root" => root}) + assert(loaded?("libfs_extension.so"), "FS did not load during construction") + payload = "ruby-runtime-poc-" * 512 + operator.write("hello.txt", payload) + assert(operator.read("hello.txt") == payload, "FS round trip failed") + assert(operator.info.fetch("scheme") == "fs", "unexpected operator scheme") + operator.close + + begin + operator.read("hello.txt") + raise "closed operator remained usable" + rescue RuntimeError => error + assert(error.message == "operator is closed", "unexpected closed-handle error") + end +end + +main = ENV.fetch("OPENDAL_RUBY_POC_MAIN") +env = {"OPENDAL_POC_REQUIRED_RUNTIME_PROTOCOL" => "2"} +_, stderr, status = Open3.capture3( + env, + RbConfig.ruby, + "-I#{main}", + "-ropendal", + "-e", + "abort 'incompatible runtime unexpectedly loaded'" +) +assert(!status.success?, "newer binding protocol unexpectedly loaded") +assert(stderr.include?("runtime protocol negotiation failed"), "missing protocol diagnostic") + +puts({ + adapter: "Magnus", + runtime_protocol: OpenDal::Runtime.runtime_protocol, + fs_lazy_load: true, + fs_round_trip: true, + closed_handle_rejected: true, + newer_protocol_rejected: true +}.inspect)
diff --git a/bindings/ruby/dynamic-extensions/run-ruby-linux.sh b/bindings/ruby/dynamic-extensions/run-ruby-linux.sh new file mode 100755 index 0000000..aa84027 --- /dev/null +++ b/bindings/ruby/dynamic-extensions/run-ruby-linux.sh
@@ -0,0 +1,70 @@ +#!/usr/bin/env sh +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to you under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -eu + +prototype_dir=$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd) +shared_dir="$prototype_dir/../../python/dynamic-extensions" +target_root="$prototype_dir/target" +runtime_target="$target_root/shared-runtime" +fs_target="$target_root/shared-fs" +adapter_target="$target_root/ruby-adapter" +stage="$target_root/ruby-stage" + +if cargo tree --locked --offline --manifest-path "$prototype_dir/Cargo.toml" \ + --package opendal-ruby-runtime-poc \ + | rg -q 'opendal-core|opendal-service-|tokio'; then + echo "Ruby adapter unexpectedly links OpenDAL core, a service, or Tokio" >&2 + exit 1 +fi + +CARGO_TARGET_DIR="$runtime_target" cargo build --release --locked --offline \ + --manifest-path "$shared_dir/Cargo.toml" --package opendal-runtime-poc +CARGO_TARGET_DIR="$fs_target" cargo build --release --locked --offline \ + --manifest-path "$shared_dir/Cargo.toml" --package fs-extension +CARGO_TARGET_DIR="$adapter_target" cargo build --release --locked --offline \ + --manifest-path "$prototype_dir/Cargo.toml" --package opendal-ruby-runtime-poc + +runtime="$runtime_target/release/libopendal_runtime_poc.so" +fs="$fs_target/release/libfs_extension.so" +adapter="$adapter_target/release/libopendal_ruby_poc.so" + +"$shared_dir/audit-elf-exports.sh" "$runtime" opendal_runtime_get_api_v1 +"$shared_dir/audit-elf-exports.sh" "$fs" opendal_service_fs_bootstrap_v1 +"$shared_dir/audit-elf-exports.sh" \ + "$adapter" Init_opendal_ruby_poc ruby_abi_version + +mkdir -p \ + "$stage/main/lib/opendal/_native" \ + "$stage/fs/lib/opendal/services/fs/_native" \ + "$stage/python/main/opendal/_native" \ + "$stage/python/fs/opendal/services/fs/_native" +cp -R "$prototype_dir/ruby/main/." "$stage/main/" +cp -R "$prototype_dir/ruby/service-fs/." "$stage/fs/" +cp "$runtime" "$stage/main/lib/opendal/_native/" +cp "$adapter" "$stage/main/lib/opendal_ruby_poc.so" +cp "$fs" "$stage/fs/lib/opendal/services/fs/_native/" +cp -R "$shared_dir/python/main/." "$stage/python/main/" +cp -R "$shared_dir/python/service-fs/." "$stage/python/fs/" +cp "$runtime" "$stage/python/main/opendal/_native/" +cp "$fs" "$stage/python/fs/opendal/services/fs/_native/" + +RUBYLIB="$stage/main/lib:$stage/fs/lib" \ +OPENDAL_RUBY_POC_MAIN="$stage/main/lib" \ + ruby "$prototype_dir/ruby/test_poc.rb" +RUBYLIB="$stage/main/lib:$stage/fs/lib" ruby "$prototype_dir/ruby/example.rb" +PYTHONPATH="$stage/python/main:$stage/python/fs" \ + python3 "$prototype_dir/ruby/python_cross_check.py"