[executor] Introduce teaclave executor and separate builtin functions
diff --git a/executor/Cargo.toml b/executor/Cargo.toml
new file mode 100644
index 0000000..b6428f4
--- /dev/null
+++ b/executor/Cargo.toml
@@ -0,0 +1,43 @@
+[package]
+name = "teaclave_executor"
+version = "0.1.0"
+authors = ["Teaclave Contributors <dev@teaclave.apache.org>"]
+description = "Teaclave executor"
+license = "Apache-2.0"
+edition = "2018"
+
+[lib]
+name = "teaclave_executor"
+crate-type = ["staticlib", "rlib"]
+
+[features]
+default = []
+mesalock_sgx = [
+ "sgx_tstd",
+ "teaclave_types/mesalock_sgx",
+ "teaclave_function/mesalock_sgx",
+]
+cov = ["sgx_cov"]
+enclave_unit_test = [
+ "teaclave_test_utils/mesalock_sgx",
+ "teaclave_runtime/mesalock_sgx"
+]
+
+[dependencies]
+log = { version = "0.4.6" }
+anyhow = { version = "1.0.26" }
+serde_json = { version = "1.0.39" }
+serde = { version = "1.0.92", features = ["derive"] }
+thiserror = { version = "1.0.9" }
+gbdt = { version = "0.1.0", features = ["input", "enable_training"] }
+rusty-machine = { version = "0.5.4" }
+itertools = { version = "0.8.0", default-features = false }
+teaclave_types = { path = "../types" }
+teaclave_crypto = { path = "../crypto" }
+teaclave_runtime = { path = "../runtime", optional = true }
+teaclave_test_utils = { path = "../tests/utils", optional = true }
+teaclave_function = { path = "../function" }
+
+sgx_cov = { version = "1.1.2", optional = true }
+sgx_tstd = { version = "1.1.2", features = ["net", "thread", "backtrace"], optional = true }
+sgx_types = { version = "1.1.2" }
diff --git a/executor/src/builtin.rs b/executor/src/builtin.rs
new file mode 100644
index 0000000..b5c37f6
--- /dev/null
+++ b/executor/src/builtin.rs
@@ -0,0 +1,57 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#[cfg(feature = "mesalock_sgx")]
+use std::prelude::v1::*;
+
+use teaclave_function::{
+ Echo, GbdtPredict, GbdtTrain, LogisticRegressionPredict, LogisticRegressionTrain,
+};
+use teaclave_types::{FunctionArguments, FunctionRuntime, TeaclaveExecutor};
+
+use anyhow::{bail, Result};
+
+#[derive(Default)]
+pub struct BuiltinFunctionExecutor;
+
+impl TeaclaveExecutor for BuiltinFunctionExecutor {
+ fn execute(
+ &self,
+ name: String,
+ arguments: FunctionArguments,
+ _payload: String,
+ runtime: FunctionRuntime,
+ ) -> Result<String> {
+ match name.as_str() {
+ Echo::NAME => Echo::new().run(arguments, runtime),
+ GbdtPredict::NAME => GbdtPredict::new().run(arguments, runtime),
+ GbdtTrain::NAME => GbdtTrain::new().run(arguments, runtime),
+ LogisticRegressionTrain::NAME => LogisticRegressionTrain::new().run(arguments, runtime),
+ LogisticRegressionPredict::NAME => {
+ LogisticRegressionPredict::new().run(arguments, runtime)
+ }
+ _ => bail!("Function not found."),
+ }
+ }
+}
+
+#[cfg(feature = "enclave_unit_test")]
+pub mod tests {
+ pub fn run_tests() -> bool {
+ true
+ }
+}
diff --git a/function/src/context.rs b/executor/src/context.rs
similarity index 99%
rename from function/src/context.rs
rename to executor/src/context.rs
index 2f90c5a..50cc3fc 100644
--- a/function/src/context.rs
+++ b/executor/src/context.rs
@@ -290,6 +290,7 @@
assert!(rtc_close_handle(f).is_ok());
assert!(rtc_close_handle(f).is_err());
+ reset_thread_context().unwrap();
}
}
diff --git a/executor/src/lib.rs b/executor/src/lib.rs
new file mode 100644
index 0000000..e9e6cb0
--- /dev/null
+++ b/executor/src/lib.rs
@@ -0,0 +1,47 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#![cfg_attr(feature = "mesalock_sgx", no_std)]
+#[cfg(feature = "mesalock_sgx")]
+extern crate sgx_tstd as std;
+
+#[cfg(feature = "mesalock_sgx")]
+use std::prelude::v1::*;
+
+#[macro_use]
+extern crate log;
+
+mod builtin;
+mod context;
+mod mesapy;
+
+pub use builtin::BuiltinFunctionExecutor;
+pub use mesapy::MesaPy;
+
+#[cfg(feature = "enclave_unit_test")]
+pub mod tests {
+ use super::*;
+ use teaclave_test_utils::check_all_passed;
+
+ pub fn run_tests() -> bool {
+ check_all_passed!(
+ context::tests::run_tests(),
+ mesapy::tests::run_tests(),
+ builtin::tests::run_tests(),
+ )
+ }
+}
diff --git a/function/src/mesapy.rs b/executor/src/mesapy.rs
similarity index 84%
rename from function/src/mesapy.rs
rename to executor/src/mesapy.rs
index d78a106..825ae53 100644
--- a/function/src/mesapy.rs
+++ b/executor/src/mesapy.rs
@@ -15,16 +15,15 @@
// specific language governing permissions and limitations
// under the License.
-#[cfg(feature = "mesalock_sgx")]
use std::prelude::v1::*;
-use std::ffi::CString;
-
use crate::context::reset_thread_context;
use crate::context::set_thread_context;
use crate::context::Context;
-use teaclave_types::{FunctionArguments, FunctionRuntime, TeaclaveFunction};
+use std::ffi::CString;
+
+use teaclave_types::{FunctionArguments, FunctionRuntime, TeaclaveExecutor};
const MAXPYBUFLEN: usize = 20480;
const MESAPY_ERROR_BUFFER_TOO_SHORT: i64 = -1i64;
@@ -41,20 +40,23 @@
}
#[derive(Default)]
-pub struct Mesapy;
+pub struct MesaPy;
-impl TeaclaveFunction for Mesapy {
- fn execute(&self, runtime: FunctionRuntime, args: FunctionArguments) -> anyhow::Result<String> {
- let script = args.get("py_payload")?.as_str();
- let py_args = args.get("py_args")?.as_str();
- let py_args: FunctionArguments = serde_json::from_str(py_args)?;
- let py_argv = py_args.into_vec();
+impl TeaclaveExecutor for MesaPy {
+ fn execute(
+ &self,
+ _name: String,
+ arguments: FunctionArguments,
+ payload: String,
+ runtime: FunctionRuntime,
+ ) -> anyhow::Result<String> {
+ let py_argv = arguments.into_vec();
let cstr_argv: Vec<_> = py_argv
.iter()
.map(|arg| CString::new(arg.as_str()).unwrap())
.collect();
- let mut script_bytes = script.to_owned().into_bytes();
+ let mut script_bytes = payload.into_bytes();
script_bytes.push(0u8);
let mut p_argv: Vec<_> = cstr_argv
@@ -104,7 +106,7 @@
}
fn test_mesapy() {
- let py_args = FunctionArguments::new(hashmap!("--name" => "Teaclave"));
+ let py_args = FunctionArguments::default();
let py_payload = r#"
def entrypoint(argv):
in_file_id = "in_f1"
@@ -165,13 +167,10 @@
let runtime = Box::new(RawIoRuntime::new(input_files, output_files));
- let func_args = FunctionArguments::new(hashmap!(
- "py_payload" => py_payload,
- "py_args" => serde_json::to_string(&py_args).unwrap()
- ));
-
- let function = Mesapy;
- let summary = function.execute(runtime, func_args).unwrap();
+ let function = MesaPy::default();
+ let summary = function
+ .execute("".to_string(), py_args, py_payload.to_string(), runtime)
+ .unwrap();
assert_eq!(summary, "");
}
}
diff --git a/function/Cargo.toml b/function/Cargo.toml
index 55d5749..9fcf244 100644
--- a/function/Cargo.toml
+++ b/function/Cargo.toml
@@ -2,7 +2,7 @@
name = "teaclave_function"
version = "0.1.0"
authors = ["Teaclave Contributors <dev@teaclave.apache.org>"]
-description = "Teaclave function"
+description = "Teaclave built-in functions."
license = "Apache-2.0"
edition = "2018"
diff --git a/function/src/echo.rs b/function/src/echo.rs
index 3cc15a5..a6de293 100644
--- a/function/src/echo.rs
+++ b/function/src/echo.rs
@@ -19,7 +19,7 @@
use std::prelude::v1::*;
use std::convert::TryFrom;
-use teaclave_types::{FunctionArguments, FunctionRuntime, TeaclaveFunction};
+use teaclave_types::{FunctionArguments, FunctionRuntime};
#[derive(Default)]
pub struct Echo;
@@ -37,11 +37,17 @@
}
}
-impl TeaclaveFunction for Echo {
- fn execute(
+impl Echo {
+ pub const NAME: &'static str = "builtin-echo";
+
+ pub fn new() -> Self {
+ Default::default()
+ }
+
+ pub fn run(
&self,
- _runtime: FunctionRuntime,
arguments: FunctionArguments,
+ _runtime: FunctionRuntime,
) -> anyhow::Result<String> {
let message = EchoArguments::try_from(arguments)?.message;
Ok(message)
@@ -60,7 +66,7 @@
}
fn test_echo() {
- let func_args = FunctionArguments::new(hashmap!(
+ let args = FunctionArguments::new(hashmap!(
"message" => "Hello Teaclave!"
));
@@ -70,7 +76,7 @@
let runtime = Box::new(RawIoRuntime::new(input_files, output_files));
let function = Echo;
- let summary = function.execute(runtime, func_args).unwrap();
+ let summary = function.run(args, runtime).unwrap();
assert_eq!(summary, "Hello Teaclave!");
}
}
diff --git a/function/src/gbdt_prediction.rs b/function/src/gbdt_predict.rs
similarity index 91%
rename from function/src/gbdt_prediction.rs
rename to function/src/gbdt_predict.rs
index 93016fe..d60651a 100644
--- a/function/src/gbdt_prediction.rs
+++ b/function/src/gbdt_predict.rs
@@ -21,7 +21,7 @@
use std::format;
use std::io::{self, BufRead, BufReader, Write};
-use teaclave_types::{FunctionArguments, FunctionRuntime, TeaclaveFunction};
+use teaclave_types::{FunctionArguments, FunctionRuntime};
use gbdt::decision_tree::Data;
use gbdt::gradient_boost::GBDT;
@@ -31,13 +31,19 @@
const OUT_RESULT: &str = "result_file";
#[derive(Default)]
-pub struct GbdtPrediction;
+pub struct GbdtPredict;
-impl TeaclaveFunction for GbdtPrediction {
- fn execute(
+impl GbdtPredict {
+ pub const NAME: &'static str = "builtin-gbdt-predict";
+
+ pub fn new() -> Self {
+ Default::default()
+ }
+
+ pub fn run(
&self,
- runtime: FunctionRuntime,
_arguments: FunctionArguments,
+ runtime: FunctionRuntime,
) -> anyhow::Result<String> {
let mut json_model = String::new();
let mut f = runtime.open_input(IN_MODEL)?;
@@ -102,7 +108,7 @@
}
fn test_gbdt_prediction() {
- let func_args = FunctionArguments::default();
+ let arguments = FunctionArguments::default();
let plain_model = "fixtures/functions/gbdt_prediction/model.txt";
let plain_data = "fixtures/functions/gbdt_prediction/test_data.txt";
@@ -123,8 +129,7 @@
let runtime = Box::new(RawIoRuntime::new(input_files, output_files));
- let function = GbdtPrediction;
- let summary = function.execute(runtime, func_args).unwrap();
+ let summary = GbdtPredict::new().run(arguments, runtime).unwrap();
assert_eq!(summary, "Predict result has 30 lines of data.");
let result = fs::read_to_string(&plain_output).unwrap();
diff --git a/function/src/gbdt_training.rs b/function/src/gbdt_train.rs
similarity index 91%
rename from function/src/gbdt_training.rs
rename to function/src/gbdt_train.rs
index 38d34a3..b887427 100644
--- a/function/src/gbdt_training.rs
+++ b/function/src/gbdt_train.rs
@@ -22,7 +22,7 @@
use std::io::{self, BufRead, BufReader, Write};
use std::convert::TryFrom;
-use teaclave_types::{FunctionArguments, FunctionRuntime, TeaclaveFunction};
+use teaclave_types::{FunctionArguments, FunctionRuntime};
use gbdt::config::Config;
use gbdt::decision_tree::Data;
@@ -32,9 +32,9 @@
const OUT_MODEL: &str = "trained_model";
#[derive(Default)]
-pub struct GbdtTraining;
+pub struct GbdtTrain;
-struct GbdtTrainingArguments {
+struct GbdtTrainArguments {
feature_size: usize,
max_depth: u32,
iterations: usize,
@@ -46,7 +46,7 @@
training_optimization_level: u8,
}
-impl TryFrom<FunctionArguments> for GbdtTrainingArguments {
+impl TryFrom<FunctionArguments> for GbdtTrainArguments {
type Error = anyhow::Error;
fn try_from(arguments: FunctionArguments) -> Result<Self, Self::Error> {
@@ -74,14 +74,20 @@
}
}
-impl TeaclaveFunction for GbdtTraining {
- fn execute(
+impl GbdtTrain {
+ pub const NAME: &'static str = "builtin-gbdt-train";
+
+ pub fn new() -> Self {
+ Default::default()
+ }
+
+ pub fn run(
&self,
- runtime: FunctionRuntime,
arguments: FunctionArguments,
+ runtime: FunctionRuntime,
) -> anyhow::Result<String> {
log::debug!("start traning...");
- let args = GbdtTrainingArguments::try_from(arguments)?;
+ let args = GbdtTrainArguments::try_from(arguments)?;
log::debug!("open input...");
// read input
@@ -165,11 +171,11 @@
use teaclave_types::*;
pub fn run_tests() -> bool {
- run_tests!(test_gbdt_training, test_gbdt_parse_training_data,)
+ run_tests!(test_gbdt_train, test_gbdt_parse_training_data,)
}
- fn test_gbdt_training() {
- let func_arguments = FunctionArguments::new(hashmap!(
+ fn test_gbdt_train() {
+ let arguments = FunctionArguments::new(hashmap!(
"feature_size" => "4",
"max_depth" => "4",
"iterations" => "100",
@@ -197,8 +203,7 @@
let runtime = Box::new(RawIoRuntime::new(input_files, output_files));
- let function = GbdtTraining;
- let summary = function.execute(runtime, func_arguments).unwrap();
+ let summary = GbdtTrain::new().run(arguments, runtime).unwrap();
assert_eq!(summary, "Trained 120 lines of data.");
let result = fs::read_to_string(&plain_output).unwrap();
diff --git a/function/src/lib.rs b/function/src/lib.rs
index a329e6e..a2c0c56 100644
--- a/function/src/lib.rs
+++ b/function/src/lib.rs
@@ -22,23 +22,17 @@
#[cfg(feature = "mesalock_sgx")]
use std::prelude::v1::*;
-#[macro_use]
-extern crate log;
-
-mod context;
mod echo;
-mod gbdt_prediction;
-mod gbdt_training;
-mod logistic_regression_prediction;
-mod logistic_regression_training;
-mod mesapy;
+mod gbdt_predict;
+mod gbdt_train;
+mod logistic_regression_predict;
+mod logistic_regression_train;
pub use echo::Echo;
-pub use gbdt_prediction::GbdtPrediction;
-pub use gbdt_training::GbdtTraining;
-pub use logistic_regression_prediction::LogitRegPrediction;
-pub use logistic_regression_training::LogitRegTraining;
-pub use mesapy::Mesapy;
+pub use gbdt_predict::GbdtPredict;
+pub use gbdt_train::GbdtTrain;
+pub use logistic_regression_predict::LogisticRegressionPredict;
+pub use logistic_regression_train::LogisticRegressionTrain;
#[cfg(feature = "enclave_unit_test")]
pub mod tests {
@@ -48,12 +42,10 @@
pub fn run_tests() -> bool {
check_all_passed!(
echo::tests::run_tests(),
- gbdt_training::tests::run_tests(),
- gbdt_prediction::tests::run_tests(),
- mesapy::tests::run_tests(),
- context::tests::run_tests(),
- logistic_regression_training::tests::run_tests(),
- logistic_regression_prediction::tests::run_tests(),
+ gbdt_train::tests::run_tests(),
+ gbdt_predict::tests::run_tests(),
+ logistic_regression_train::tests::run_tests(),
+ logistic_regression_predict::tests::run_tests(),
)
}
}
diff --git a/function/src/logistic_regression_prediction.rs b/function/src/logistic_regression_predict.rs
similarity index 88%
rename from function/src/logistic_regression_prediction.rs
rename to function/src/logistic_regression_predict.rs
index 344be1c..ec2dddc 100644
--- a/function/src/logistic_regression_prediction.rs
+++ b/function/src/logistic_regression_predict.rs
@@ -21,7 +21,7 @@
use std::format;
use std::io::{self, BufRead, BufReader, Write};
-use teaclave_types::{FunctionArguments, FunctionRuntime, TeaclaveFunction};
+use teaclave_types::{FunctionArguments, FunctionRuntime};
use rusty_machine::learning::logistic_reg::LogisticRegressor;
use rusty_machine::learning::optim::grad_desc::GradientDesc;
@@ -33,13 +33,19 @@
const RESULT: &str = "result_file";
#[derive(Default)]
-pub struct LogitRegPrediction;
+pub struct LogisticRegressionPredict;
-impl TeaclaveFunction for LogitRegPrediction {
- fn execute(
+impl LogisticRegressionPredict {
+ pub const NAME: &'static str = "builtin-logistic-regression-predict";
+
+ pub fn new() -> Self {
+ Default::default()
+ }
+
+ pub fn run(
&self,
- runtime: FunctionRuntime,
_arguments: FunctionArguments,
+ runtime: FunctionRuntime,
) -> anyhow::Result<String> {
let mut model_json = String::new();
let mut f = runtime.open_input(MODEL_FILE)?;
@@ -109,11 +115,11 @@
use teaclave_types::*;
pub fn run_tests() -> bool {
- run_tests!(test_logistic_regression_prediction)
+ run_tests!(test_logistic_regression_predict)
}
- fn test_logistic_regression_prediction() {
- let func_args = FunctionArguments::default();
+ fn test_logistic_regression_predict() {
+ let arguments = FunctionArguments::default();
let base = Path::new("fixtures/functions/logistic_regression_prediction");
let model = base.join("model.txt");
@@ -135,8 +141,9 @@
let runtime = Box::new(RawIoRuntime::new(input_files, output_files));
- let function = LogitRegPrediction;
- let summary = function.execute(runtime, func_args).unwrap();
+ let summary = LogisticRegressionPredict::new()
+ .run(arguments, runtime)
+ .unwrap();
assert_eq!(summary, "Predicted 5 lines of data.");
let result = fs::read_to_string(&plain_output).unwrap();
diff --git a/function/src/logistic_regression_training.rs b/function/src/logistic_regression_train.rs
similarity index 86%
rename from function/src/logistic_regression_training.rs
rename to function/src/logistic_regression_train.rs
index 09214ee..5df3bbe 100644
--- a/function/src/logistic_regression_training.rs
+++ b/function/src/logistic_regression_train.rs
@@ -22,7 +22,7 @@
use std::format;
use std::io::{self, BufRead, BufReader, Write};
-use teaclave_types::{FunctionArguments, FunctionRuntime, TeaclaveFunction};
+use teaclave_types::{FunctionArguments, FunctionRuntime};
use rusty_machine::learning::logistic_reg::LogisticRegressor;
use rusty_machine::learning::optim::grad_desc::GradientDesc;
@@ -33,15 +33,15 @@
const OUT_MODEL_FILE: &str = "model_file";
#[derive(Default)]
-pub struct LogitRegTraining;
+pub struct LogisticRegressionTrain;
-struct LogitRegTrainingArguments {
+struct LogisticRegressionTrainArguments {
alg_alpha: f64,
alg_iters: usize,
feature_size: usize,
}
-impl TryFrom<FunctionArguments> for LogitRegTrainingArguments {
+impl TryFrom<FunctionArguments> for LogisticRegressionTrainArguments {
type Error = anyhow::Error;
fn try_from(arguments: FunctionArguments) -> Result<Self, Self::Error> {
@@ -57,13 +57,19 @@
}
}
-impl TeaclaveFunction for LogitRegTraining {
- fn execute(
+impl LogisticRegressionTrain {
+ pub const NAME: &'static str = "builtin-logistic-regression-train";
+
+ pub fn new() -> Self {
+ Default::default()
+ }
+
+ pub fn run(
&self,
- runtime: FunctionRuntime,
arguments: FunctionArguments,
+ runtime: FunctionRuntime,
) -> anyhow::Result<String> {
- let args = LogitRegTrainingArguments::try_from(arguments)?;
+ let args = LogisticRegressionTrainArguments::try_from(arguments)?;
let input = runtime.open_input(TRAINING_DATA)?;
let (flattend_features, targets) = parse_training_data(input, args.feature_size)?;
@@ -127,11 +133,11 @@
use teaclave_types::*;
pub fn run_tests() -> bool {
- run_tests!(test_logistic_regression_training)
+ run_tests!(test_logistic_regression_train)
}
- fn test_logistic_regression_training() {
- let func_args = FunctionArguments::new(hashmap! {
+ fn test_logistic_regression_train() {
+ let arguments = FunctionArguments::new(hashmap! {
"alg_alpha" => "0.3",
"alg_iters" => "100",
"feature_size" => "30"
@@ -154,8 +160,9 @@
let runtime = Box::new(RawIoRuntime::new(input_files, output_files));
- let function = LogitRegTraining;
- let summary = function.execute(runtime, func_args).unwrap();
+ let summary = LogisticRegressionTrain::new()
+ .run(arguments, runtime)
+ .unwrap();
assert_eq!(summary, "Trained 100 lines of data.");
let _result = fs::read_to_string(&plain_output).unwrap();
diff --git a/services/execution/enclave/src/lib.rs b/services/execution/enclave/src/lib.rs
index babefa8..af99909 100644
--- a/services/execution/enclave/src/lib.rs
+++ b/services/execution/enclave/src/lib.rs
@@ -115,7 +115,7 @@
run_tests!(
ocall::tests::test_handle_file_request,
service::tests::test_invoke_echo,
- service::tests::test_invoke_gbdt_training,
+ service::tests::test_invoke_gbdt_train,
task_file_manager::tests::test_input,
)
}
diff --git a/services/execution/enclave/src/service.rs b/services/execution/enclave/src/service.rs
index 1b73311..a0e419d 100644
--- a/services/execution/enclave/src/service.rs
+++ b/services/execution/enclave/src/service.rs
@@ -64,7 +64,7 @@
let staged_task = match self.pull_task() {
Ok(staged_task) => staged_task,
Err(e) => {
- log::error!("PullTask Error: {:?}", e);
+ log::warn!("PullTask Error: {:?}", e);
continue;
}
};
@@ -154,8 +154,9 @@
let staged_function = StagedFunction::new()
.executor_type(task.executor_type)
.executor(task.executor)
- .payload(function_payload)
+ .name(&task.function_name)
.arguments(task.function_arguments.clone())
+ .payload(function_payload)
.input_files(input_files)
.output_files(output_files)
.runtime_name("default");
@@ -178,7 +179,8 @@
let task_id = Uuid::new_v4();
let staged_task = StagedTask::new()
.task_id(task_id)
- .executor(Executor::Echo)
+ .executor(Executor::Builtin)
+ .function_name("builtin-echo")
.function_arguments(hashmap!("message" => "Hello, Teaclave!"));
let file_mgr = TaskFileManager::new(
@@ -198,7 +200,7 @@
assert_eq!(result.unwrap(), "Hello, Teaclave!");
}
- pub fn test_invoke_gbdt_training() {
+ pub fn test_invoke_gbdt_train() {
let task_id = Uuid::new_v4();
let function_arguments = FunctionArguments::new(hashmap!(
"feature_size" => "4",
@@ -232,7 +234,8 @@
let staged_task = StagedTask::new()
.task_id(task_id)
- .executor(Executor::GbdtTraining)
+ .executor(Executor::Builtin)
+ .function_name("builtin-gbdt-train")
.function_arguments(function_arguments)
.input_data(input_data)
.output_data(output_data);
diff --git a/services/proto/src/teaclave_frontend_service.rs b/services/proto/src/teaclave_frontend_service.rs
index f1bf80c..9ca53ff 100644
--- a/services/proto/src/teaclave_frontend_service.rs
+++ b/services/proto/src/teaclave_frontend_service.rs
@@ -223,7 +223,11 @@
impl RegisterFunctionRequest {
pub fn new() -> Self {
- Self::default()
+ Self {
+ executor_type: ExecutorType::Builtin,
+ public: true,
+ ..Default::default()
+ }
}
pub fn name(self, name: impl ToString) -> Self {
diff --git a/tests/functional/enclave/src/end_to_end/native_echo.rs b/tests/functional/enclave/src/end_to_end/builtin_echo.rs
similarity index 94%
rename from tests/functional/enclave/src/end_to_end/native_echo.rs
rename to tests/functional/enclave/src/end_to_end/builtin_echo.rs
index bd67bc0..5d39ba3 100644
--- a/tests/functional/enclave/src/end_to_end/native_echo.rs
+++ b/tests/functional/enclave/src/end_to_end/builtin_echo.rs
@@ -29,10 +29,8 @@
// Register Function
let request = RegisterFunctionRequest::new()
- .name("native_echo_demo")
+ .name("builtin-echo")
.description("Native Echo Function")
- .executor_type(ExecutorType::Native)
- .public(true)
.arguments(vec!["message"]);
let response = client.register_function(request).unwrap();
@@ -44,7 +42,7 @@
let request = CreateTaskRequest::new()
.function_id(function_id)
.function_arguments(hashmap!("message" => "Hello From Teaclave!"))
- .executor(Executor::Echo);
+ .executor(Executor::Builtin);
let response = client.create_task(request).unwrap();
diff --git a/tests/functional/enclave/src/end_to_end/native_gbdt_training.rs b/tests/functional/enclave/src/end_to_end/builtin_gbdt_train.rs
similarity index 96%
rename from tests/functional/enclave/src/end_to_end/native_gbdt_training.rs
rename to tests/functional/enclave/src/end_to_end/builtin_gbdt_train.rs
index 3747596..86258ee 100644
--- a/tests/functional/enclave/src/end_to_end/native_gbdt_training.rs
+++ b/tests/functional/enclave/src/end_to_end/builtin_gbdt_train.rs
@@ -62,10 +62,8 @@
// Register Function
let request = RegisterFunctionRequest::new()
- .name("native_gbdt_training_demo")
+ .name("builtin-gbdt-train")
.description("Native Gbdt Training Function")
- .executor_type(ExecutorType::Native)
- .public(true)
.arguments(fn_args)
.inputs(vec![fn_input])
.outputs(vec![fn_output]);
@@ -106,7 +104,7 @@
function_id: &ExternalID,
) -> ExternalID {
let request = CreateTaskRequest::new()
- .executor(Executor::GbdtTraining)
+ .executor(Executor::Builtin)
.function_id(function_id.clone())
.function_arguments(hashmap!(
"feature_size" => "4",
diff --git a/tests/functional/enclave/src/end_to_end/mod.rs b/tests/functional/enclave/src/end_to_end/mod.rs
index 64a7020..e6f5502 100644
--- a/tests/functional/enclave/src/end_to_end/mod.rs
+++ b/tests/functional/enclave/src/end_to_end/mod.rs
@@ -21,10 +21,10 @@
use teaclave_types::*;
use url::Url;
+mod builtin_echo;
+mod builtin_gbdt_train;
mod mesapy_data_fusion;
mod mesapy_echo;
-mod native_echo;
-mod native_gbdt_training;
fn get_task(client: &mut TeaclaveFrontendClient, task_id: &ExternalID) -> GetTaskResponse {
let request = GetTaskRequest::new(task_id.clone());
diff --git a/tests/functional/enclave/src/execution_service.rs b/tests/functional/enclave/src/execution_service.rs
index 9590daa..2412197 100644
--- a/tests/functional/enclave/src/execution_service.rs
+++ b/tests/functional/enclave/src/execution_service.rs
@@ -37,7 +37,8 @@
let staged_task = StagedTask::new()
.task_id(task_id)
.function_id(function_id.clone())
- .executor(Executor::Echo)
+ .function_name("builtin-echo")
+ .executor(Executor::Builtin)
.function_arguments(hashmap!(
"message" => "Hello, Teaclave Tests!"
));
diff --git a/tests/functional/enclave/src/scheduler_service.rs b/tests/functional/enclave/src/scheduler_service.rs
index 24c4f7f..226f273 100644
--- a/tests/functional/enclave/src/scheduler_service.rs
+++ b/tests/functional/enclave/src/scheduler_service.rs
@@ -29,8 +29,9 @@
let function_id = Uuid::new_v4();
let staged_task = StagedTask::new()
.task_id(Uuid::new_v4())
+ .function_name("builtin-echo")
.function_id(function_id.clone())
- .executor(Executor::Echo);
+ .executor(Executor::Builtin);
let mut storage_client = get_storage_client();
let enqueue_request = EnqueueRequest::new(
@@ -60,8 +61,9 @@
let staged_task = StagedTask::new()
.task_id(task_id.clone())
+ .function_name("builtin-echo")
.function_id(function_id)
- .executor(Executor::Echo);
+ .executor(Executor::Builtin);
let mut storage_client = get_storage_client();
let enqueue_request = EnqueueRequest::new(
diff --git a/tests/integration/enclave/src/teaclave_worker.rs b/tests/integration/enclave/src/teaclave_worker.rs
index d329941..09b9f5d 100644
--- a/tests/integration/enclave/src/teaclave_worker.rs
+++ b/tests/integration/enclave/src/teaclave_worker.rs
@@ -14,6 +14,7 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+
use std::prelude::v1::*;
use teaclave_crypto::TeaclaveFile128Key;
@@ -54,8 +55,9 @@
"trained_model" => output_info.clone()));
let staged_function = StagedFunction::new()
- .executor_type(ExecutorType::Native)
- .executor(Executor::GbdtTraining)
+ .executor_type(ExecutorType::Builtin)
+ .executor(Executor::Builtin)
+ .name("builtin-gbdt-train")
.arguments(arguments)
.input_files(input_files)
.output_files(output_files)
@@ -63,10 +65,6 @@
let worker = Worker::default();
- let capability = worker.get_capability();
- assert!(capability.runtimes.contains("default"));
- assert!(capability.functions.contains("native-gbdt_training"));
-
let summary = worker.invoke_function(staged_function).unwrap();
assert_eq!(summary, "Trained 120 lines of data.");
diff --git a/tests/unit/enclave/Cargo.toml b/tests/unit/enclave/Cargo.toml
index 5dc8e27..8ebba92 100644
--- a/tests/unit/enclave/Cargo.toml
+++ b/tests/unit/enclave/Cargo.toml
@@ -38,6 +38,8 @@
"teaclave_scheduler_service_enclave/enclave_unit_test",
"teaclave_worker/mesalock_sgx",
"teaclave_worker/enclave_unit_test",
+ "teaclave_executor/mesalock_sgx",
+ "teaclave_executor/enclave_unit_test",
"teaclave_function/mesalock_sgx",
"teaclave_function/enclave_unit_test",
"teaclave_runtime/mesalock_sgx",
@@ -61,6 +63,7 @@
teaclave_scheduler_service_enclave = { path = "../../../services/scheduler/enclave" }
teaclave_worker = { path = "../../../worker" }
+teaclave_executor = { path = "../../../executor" }
teaclave_function = { path = "../../../function" }
teaclave_runtime = { path = "../../../runtime" }
rusty-leveldb = { path = "../../../common/rusty_leveldb_sgx", default-features = false, optional = true }
diff --git a/tests/unit/enclave/src/lib.rs b/tests/unit/enclave/src/lib.rs
index 008f002..3e327e4 100644
--- a/tests/unit/enclave/src/lib.rs
+++ b/tests/unit/enclave/src/lib.rs
@@ -40,6 +40,7 @@
teaclave_authentication_service_enclave::tests::run_tests(),
teaclave_worker::tests::run_tests(),
teaclave_runtime::tests::run_tests(),
+ teaclave_executor::tests::run_tests(),
teaclave_function::tests::run_tests(),
teaclave_types::tests::run_tests(),
teaclave_crypto::tests::run_tests(),
diff --git a/types/src/staged_function.rs b/types/src/staged_function.rs
index 03a46b9..47f3bd9 100644
--- a/types/src/staged_function.rs
+++ b/types/src/staged_function.rs
@@ -157,13 +157,14 @@
#[derive(Debug, Default)]
pub struct StagedFunction {
- pub executor: Executor,
- pub payload: String,
+ pub name: String,
pub arguments: FunctionArguments,
+ pub payload: String,
pub input_files: StagedFiles,
pub output_files: StagedFiles,
- pub runtime_name: String,
pub executor_type: ExecutorType,
+ pub executor: Executor,
+ pub runtime_name: String,
}
impl StagedFunction {
@@ -171,6 +172,13 @@
Self::default()
}
+ pub fn name(self, name: impl ToString) -> Self {
+ Self {
+ name: name.to_string(),
+ ..self
+ }
+ }
+
pub fn executor(self, executor: Executor) -> Self {
Self { executor, ..self }
}
diff --git a/types/src/staged_task.rs b/types/src/staged_task.rs
index c2bf26e..71369c8 100644
--- a/types/src/staged_task.rs
+++ b/types/src/staged_task.rs
@@ -155,8 +155,9 @@
pub function_id: Uuid,
pub executor: Executor,
pub executor_type: ExecutorType,
- pub function_payload: Vec<u8>,
+ pub function_name: String,
pub function_arguments: FunctionArguments,
+ pub function_payload: Vec<u8>,
pub input_data: FunctionInputFiles,
pub output_data: FunctionOutputFiles,
}
@@ -191,9 +192,9 @@
Self { executor, ..self }
}
- pub fn function_payload(self, function_payload: Vec<u8>) -> Self {
+ pub fn function_name(self, name: impl ToString) -> Self {
Self {
- function_payload,
+ function_name: name.to_string(),
..self
}
}
@@ -205,6 +206,13 @@
}
}
+ pub fn function_payload(self, function_payload: Vec<u8>) -> Self {
+ Self {
+ function_payload,
+ ..self
+ }
+ }
+
pub fn input_data(self, input_data: impl Into<FunctionInputFiles>) -> Self {
Self {
input_data: input_data.into(),
diff --git a/types/src/task.rs b/types/src/task.rs
index c6f3090..6061d38 100644
--- a/types/src/task.rs
+++ b/types/src/task.rs
@@ -502,6 +502,7 @@
executor: self.executor,
executor_type: function.executor_type,
function_id: function.id,
+ function_name: function.name,
function_payload: function.payload,
function_arguments,
input_data: input_map.into(),
diff --git a/types/src/worker.rs b/types/src/worker.rs
index eebe592..aa9ced6 100644
--- a/types/src/worker.rs
+++ b/types/src/worker.rs
@@ -15,7 +15,7 @@
// specific language governing permissions and limitations
// under the License.
-use crate::{FunctionArguments, OutputsTags};
+use crate::{FunctionArguments, FunctionRuntime, OutputsTags};
use serde::{Deserialize, Serialize};
use std::collections::HashSet;
use std::convert::TryInto;
@@ -27,23 +27,25 @@
fn create_output(&self, identifier: &str) -> anyhow::Result<Box<dyn io::Write>>;
}
-pub trait TeaclaveFunction {
+pub trait TeaclaveExecutor {
fn execute(
&self,
- runtime: Box<dyn TeaclaveRuntime + Send + Sync>,
- args: FunctionArguments,
+ name: String,
+ arguments: FunctionArguments,
+ payload: String,
+ runtime: FunctionRuntime,
) -> anyhow::Result<String>;
}
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Deserialize, Serialize)]
pub enum ExecutorType {
- Native,
+ Builtin,
Python,
}
impl std::default::Default for ExecutorType {
fn default() -> Self {
- ExecutorType::Native
+ ExecutorType::Builtin
}
}
@@ -53,7 +55,7 @@
fn try_from(selector: &str) -> anyhow::Result<Self> {
let executor_type = match selector {
"python" => ExecutorType::Python,
- "native" | "platform" => ExecutorType::Native,
+ "builtin" => ExecutorType::Builtin,
_ => anyhow::bail!("Invalid executor type: {}", selector),
};
Ok(executor_type)
@@ -77,7 +79,7 @@
impl std::fmt::Display for ExecutorType {
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
- ExecutorType::Native => write!(f, "native"),
+ ExecutorType::Builtin => write!(f, "builtin"),
ExecutorType::Python => write!(f, "python"),
}
}
@@ -86,11 +88,7 @@
#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash, Deserialize, Serialize)]
pub enum Executor {
MesaPy,
- GbdtTraining,
- GbdtPrediction,
- LogitRegTraining,
- LogitRegPrediction,
- Echo,
+ Builtin,
}
impl std::default::Default for Executor {
@@ -105,11 +103,7 @@
fn try_from(selector: &str) -> anyhow::Result<Self> {
let executor = match selector {
"mesapy" => Executor::MesaPy,
- "echo" => Executor::Echo,
- "gbdt_training" => Executor::GbdtTraining,
- "gbdt_prediction" => Executor::GbdtPrediction,
- "logistic_regression_training" => Executor::LogitRegTraining,
- "logistic_regression_prediction" => Executor::LogitRegPrediction,
+ "builtin" => Executor::Builtin,
_ => anyhow::bail!("Unsupported executor: {}", selector),
};
Ok(executor)
@@ -128,11 +122,7 @@
fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
match self {
Executor::MesaPy => write!(f, "mesapy"),
- Executor::Echo => write!(f, "echo"),
- Executor::GbdtTraining => write!(f, "gbdt_training"),
- Executor::GbdtPrediction => write!(f, "gbdt_prediction"),
- Executor::LogitRegTraining => write!(f, "logistic_regression_training"),
- Executor::LogitRegPrediction => write!(f, "logistic_regression_prediction"),
+ Executor::Builtin => write!(f, "builtin"),
}
}
}
@@ -140,7 +130,7 @@
#[derive(Debug)]
pub struct WorkerCapability {
pub runtimes: HashSet<String>,
- pub functions: HashSet<String>,
+ pub executors: HashSet<String>,
}
#[derive(Debug, Default)]
@@ -152,11 +142,8 @@
#[cfg(feature = "enclave_unit_test")]
pub mod tests {
use super::*;
- //use crate::unit_tests;
- //use crate::unittest::*;
pub fn run_tests() -> bool {
- //unit_tests!()
true
}
}
diff --git a/worker/Cargo.toml b/worker/Cargo.toml
index d551ac9..fe73855 100644
--- a/worker/Cargo.toml
+++ b/worker/Cargo.toml
@@ -15,7 +15,7 @@
mesalock_sgx = [
"sgx_tstd",
"teaclave_types/mesalock_sgx",
- "teaclave_function/mesalock_sgx",
+ "teaclave_executor/mesalock_sgx",
"teaclave_runtime/mesalock_sgx"
]
cov = ["sgx_cov"]
@@ -27,7 +27,7 @@
serde_json = { version = "1.0.39" }
thiserror = { version = "1.0.9" }
teaclave_types = { path = "../types" }
-teaclave_function = { path = "../function" }
+teaclave_executor = { path = "../executor" }
teaclave_runtime = { path = "../runtime" }
teaclave_test_utils = { path = "../tests/utils", optional = true }
diff --git a/worker/src/worker.rs b/worker/src/worker.rs
index c08e7cc..c94d64a 100644
--- a/worker/src/worker.rs
+++ b/worker/src/worker.rs
@@ -21,74 +21,72 @@
use std::collections::HashMap;
use std::format;
-use teaclave_types::{
- hashmap, Executor, ExecutorType, FunctionArguments, StagedFiles, StagedFunction,
- WorkerCapability,
-};
+use teaclave_types::{Executor, ExecutorType, StagedFiles, StagedFunction};
-use teaclave_function as function;
-use teaclave_runtime as runtime;
-use teaclave_types::{TeaclaveFunction, TeaclaveRuntime};
+use teaclave_executor::{BuiltinFunctionExecutor, MesaPy};
+use teaclave_runtime::{DefaultRuntime, RawIoRuntime};
+use teaclave_types::{TeaclaveExecutor, TeaclaveRuntime};
-macro_rules! register_functions{
- ($(($executor_type: expr, $executor_name: expr) => $fn_type: ty,)*) => {{
- let mut functions: HashMap<(ExecutorType, Executor), FunctionBuilder> = HashMap::new();
- $(
- functions.insert(
- ($executor_type, $executor_name),
- Box::new(|| Box::new(<$fn_type>::default())),
- );
- )*
- functions
- }}
-}
+type BoxedTeaclaveExecutor = Box<dyn TeaclaveExecutor + Send + Sync>;
+type BoxedTeaclaveRuntime = Box<dyn TeaclaveRuntime + Send + Sync>;
+type ExecutorBuilder = fn() -> BoxedTeaclaveExecutor;
+type RuntimeBuilder = fn(StagedFiles, StagedFiles) -> BoxedTeaclaveRuntime;
pub struct Worker {
runtimes: HashMap<String, RuntimeBuilder>,
- functions: HashMap<(ExecutorType, Executor), FunctionBuilder>,
+ executors: HashMap<(ExecutorType, Executor), ExecutorBuilder>,
+}
+
+impl Default for Worker {
+ fn default() -> Self {
+ let mut worker = Worker::new();
+
+ // Register supported runtimes
+ worker.register_runtime("default", |input, output| {
+ Box::new(DefaultRuntime::new(input, output))
+ });
+
+ #[cfg(test_mode)]
+ worker.register_runtime("raw-io", |input, output| {
+ Box::new(RawIoRuntime::new(input, output))
+ });
+
+ // Register supported executors
+ worker.register_executor((ExecutorType::Python, Executor::MesaPy), || {
+ Box::new(MesaPy::default())
+ });
+ worker.register_executor((ExecutorType::Builtin, Executor::Builtin), || {
+ Box::new(BuiltinFunctionExecutor::default())
+ });
+
+ worker
+ }
}
impl Worker {
- pub fn default() -> Worker {
- Worker {
- functions: register_functions!(
- (ExecutorType::Python, Executor::MesaPy) => function::Mesapy,
- (ExecutorType::Native, Executor::Echo) => function::Echo,
- (ExecutorType::Native, Executor::GbdtTraining) => function::GbdtTraining,
- (ExecutorType::Native, Executor::GbdtPrediction) => function::GbdtPrediction,
- (ExecutorType::Native, Executor::LogitRegTraining) => function::LogitRegTraining,
- (ExecutorType::Native, Executor::LogitRegPrediction) => function::LogitRegPrediction,
- ),
- runtimes: setup_runtimes(),
+ pub fn new() -> Self {
+ Self {
+ runtimes: HashMap::new(),
+ executors: HashMap::new(),
}
}
- pub fn invoke_function(&self, staged_function: StagedFunction) -> anyhow::Result<String> {
- let function =
- self.get_function(staged_function.executor_type, staged_function.executor)?;
+ pub fn register_runtime(&mut self, name: impl ToString, builder: RuntimeBuilder) {
+ self.runtimes.insert(name.to_string(), builder);
+ }
+
+ pub fn register_executor(&mut self, key: (ExecutorType, Executor), builder: ExecutorBuilder) {
+ self.executors.insert(key, builder);
+ }
+
+ pub fn invoke_function(&self, function: StagedFunction) -> anyhow::Result<String> {
+ let executor = self.get_executor(function.executor_type, function.executor)?;
let runtime = self.get_runtime(
- &staged_function.runtime_name,
- staged_function.input_files,
- staged_function.output_files,
+ &function.runtime_name,
+ function.input_files,
+ function.output_files,
)?;
- let unified_args = prepare_arguments(
- staged_function.executor_type,
- staged_function.arguments,
- staged_function.payload,
- )?;
- function.execute(runtime, unified_args)
- }
-
- pub fn get_capability(&self) -> WorkerCapability {
- WorkerCapability {
- runtimes: self.runtimes.keys().cloned().collect(),
- functions: self
- .functions
- .keys()
- .cloned()
- .map(|(exec_type, exec_name)| make_function_identifier(exec_type, exec_name))
- .collect(),
- }
+ executor.execute(function.name, function.arguments, function.payload, runtime)
}
fn get_runtime(
@@ -96,7 +94,7 @@
name: &str,
input_files: StagedFiles,
output_files: StagedFiles,
- ) -> anyhow::Result<Box<dyn TeaclaveRuntime + Send + Sync>> {
+ ) -> anyhow::Result<BoxedTeaclaveRuntime> {
let build_runtime = self
.runtimes
.get(name)
@@ -106,79 +104,19 @@
Ok(runtime)
}
- fn get_function(
+ fn get_executor(
&self,
exec_type: ExecutorType,
exec_name: Executor,
- ) -> anyhow::Result<Box<dyn TeaclaveFunction + Send + Sync>> {
+ ) -> anyhow::Result<BoxedTeaclaveExecutor> {
let identifier = (exec_type, exec_name);
- let build_function = self
- .functions
+ let build_executor = self
+ .executors
.get(&identifier)
.ok_or_else(|| anyhow::anyhow!(format!("function not available: {:?}", identifier)))?;
- let function = build_function();
- Ok(function)
+ let executor = build_executor();
+
+ Ok(executor)
}
}
-
-fn make_function_identifier(exec_type: ExecutorType, exec_name: Executor) -> String {
- format!("{}-{}", exec_type, exec_name)
-}
-
-fn setup_runtimes() -> HashMap<String, RuntimeBuilder> {
- let mut runtimes: HashMap<String, RuntimeBuilder> = HashMap::new();
- runtimes.insert(
- "default".to_string(),
- Box::new(|input_files, output_files| {
- Box::new(runtime::DefaultRuntime::new(input_files, output_files))
- }),
- );
- #[cfg(test_mode)]
- runtimes.insert(
- "raw-io".to_string(),
- Box::new(|input_files, output_files| {
- Box::new(runtime::RawIoRuntime::new(input_files, output_files))
- }),
- );
-
- runtimes
-}
-
-// Native functions (ExecutorType::Native) are not allowed to have function payload.
-// Script engines like Mesapy (ExecutorType::Python) must have script payload.
-// We assume that the script engines would take the script payload and
-// script arguments from the wrapped argument.
-fn prepare_arguments(
- executor_type: ExecutorType,
- function_arguments: FunctionArguments,
- function_payload: String,
-) -> anyhow::Result<FunctionArguments> {
- let unified_args = match executor_type {
- ExecutorType::Native => {
- anyhow::ensure!(
- function_payload.is_empty(),
- "Native function payload should be empty!"
- );
- function_arguments
- }
- ExecutorType::Python => {
- anyhow::ensure!(
- !function_payload.is_empty(),
- "Python function payload must not be empty!"
- );
- let req_args = serde_json::to_string(&function_arguments)?;
- let wrap_args = hashmap!(
- "py_payload" => function_payload,
- "py_args" => req_args,
- );
- FunctionArguments::new(wrap_args)
- }
- };
-
- Ok(unified_args)
-}
-
-type FunctionBuilder = Box<dyn Fn() -> Box<dyn TeaclaveFunction + Send + Sync> + Send + Sync>;
-type RuntimeBuilder =
- Box<dyn Fn(StagedFiles, StagedFiles) -> Box<dyn TeaclaveRuntime + Send + Sync> + Send + Sync>;
