fix: Feature of services-hdfs not handled correctly (#264)
* fix: Feature of services-hdfs not handled correctly
Signed-off-by: Xuanwo <github@xuanwo.io>
* Add docs for behavior test
Signed-off-by: Xuanwo <github@xuanwo.io>
* Format toml
Signed-off-by: Xuanwo <github@xuanwo.io>
* Fix s3 tests not ported correctly
Signed-off-by: Xuanwo <github@xuanwo.io>
diff --git a/.github/workflows/service_test_azblob.yml b/.github/workflows/service_test_azblob.yml
index e904d8a..464e6f4 100644
--- a/.github/workflows/service_test_azblob.yml
+++ b/.github/workflows/service_test_azblob.yml
@@ -27,7 +27,7 @@
- name: Test
shell: bash
- run: cargo test azblob --features compress,retry -- --nocapture
+ run: cargo test azblob --features compress,retry,testing -- --nocapture
env:
RUST_BACKTRACE: full
RUST_LOG: debug
diff --git a/.github/workflows/service_test_fs.yml b/.github/workflows/service_test_fs.yml
index 53c06d7..f5aa185 100644
--- a/.github/workflows/service_test_fs.yml
+++ b/.github/workflows/service_test_fs.yml
@@ -18,7 +18,7 @@
- uses: actions/checkout@v3
- name: Test
shell: bash
- run: cargo test fs --features compress,retry -- --nocapture
+ run: cargo test fs --features compress,retry,testing -- --nocapture
env:
RUST_BACKTRACE: full
RUST_LOG: debug
diff --git a/.github/workflows/service_test_hdfs.yml b/.github/workflows/service_test_hdfs.yml
index a382ee3..4300e22 100644
--- a/.github/workflows/service_test_hdfs.yml
+++ b/.github/workflows/service_test_hdfs.yml
@@ -33,7 +33,7 @@
- name: Test
shell: bash
- run: cargo test hdfs --features compress,retry,services-hdfs -- --nocapture
+ run: cargo test hdfs --features compress,retry,testing,services-hdfs -- --nocapture
env:
RUST_BACKTRACE: full
RUST_LOG: debug
@@ -69,7 +69,7 @@
- name: Test
shell: bash
- run: cargo test hdfs --features compress,retry,services-hdfs -- --nocapture
+ run: cargo test hdfs --features compress,retry,testing,services-hdfs -- --nocapture
env:
RUST_BACKTRACE: full
RUST_LOG: debug
diff --git a/.github/workflows/service_test_memory.yml b/.github/workflows/service_test_memory.yml
index a06386b..a2cc71d 100644
--- a/.github/workflows/service_test_memory.yml
+++ b/.github/workflows/service_test_memory.yml
@@ -18,7 +18,7 @@
- uses: actions/checkout@v3
- name: Test
shell: bash
- run: cargo test memory --features compress,retry -- --nocapture
+ run: cargo test memory --features compress,retry,testing -- --nocapture
env:
RUST_BACKTRACE: full
RUST_LOG: debug
diff --git a/.github/workflows/service_test_s3.yml b/.github/workflows/service_test_s3.yml
index 776870e..c39a78b 100644
--- a/.github/workflows/service_test_s3.yml
+++ b/.github/workflows/service_test_s3.yml
@@ -13,7 +13,7 @@
- uses: actions/checkout@v3
- name: Test
shell: bash
- run: cargo test s3 --features compress,retry -- --nocapture
+ run: cargo test s3 --features compress,retry,testing -- --nocapture
env:
RUST_BACKTRACE: full
RUST_LOG: debug
@@ -30,7 +30,7 @@
- uses: actions/checkout@v3
- name: Test
shell: bash
- run: cargo test s3 --features compress,retry -- --nocapture
+ run: cargo test s3 --features compress,retry,testing -- --nocapture
env:
RUST_BACKTRACE: full
RUST_LOG: debug
@@ -68,7 +68,7 @@
- name: Test
shell: bash
- run: cargo test s3 --features compress,retry -- --nocapture
+ run: cargo test s3 --features compress,retry,testing -- --nocapture
env:
RUST_BACKTRACE: full
RUST_LOG: debug
@@ -103,7 +103,7 @@
- name: Test
shell: bash
- run: cargo test s3 --features compress,retry -- --nocapture
+ run: cargo test s3 --features compress,retry,testing -- --nocapture
env:
RUST_BACKTRACE: full
RUST_LOG: debug
diff --git a/Cargo.toml b/Cargo.toml
index 1eb307b..5df2655 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -15,10 +15,8 @@
[features]
compress = ["async-compression"]
retry = ["backon"]
-services-hdfs = ["hdrs", "opendal_test/services-hdfs"]
-
-[workspace]
-members = ["opendal_test"]
+services-hdfs = ["hdrs"]
+testing = ["uuid"]
[lib]
bench = false
@@ -42,6 +40,7 @@
backon = { version = "0.0.2", optional = true }
base64 = "0.13.0"
bytes = "1.1.0"
+dotenv = { version = "0.15.0", optional = true }
futures = { version = "0.3.21", features = ["alloc"] }
hdrs = { version = "0.1.3", optional = true, features = ["futures-io"] }
http = "0.2.6"
@@ -62,9 +61,11 @@
thiserror = "1.0.30"
time = "0.3.9"
tokio = { version = "1.17.0", features = ["full"] }
+uuid = { version = "1.0.0", optional = true, features = ["serde", "v4"] }
[dev-dependencies]
anyhow = "1.0.56"
+cfg-if = "1.0.0"
criterion = { version = "0.3.5", features = [
"async",
"async_tokio",
@@ -74,7 +75,6 @@
env_logger = "0.9.0"
itertools = "0.10.3"
num-traits = "0.2.14"
-opendal_test = { path = "./opendal_test" }
paste = "1.0.7"
rand = "0.8.5"
serde_json = "1.0.79"
diff --git a/benches/ops/utils.rs b/benches/ops/utils.rs
index 7633433..1860e5e 100644
--- a/benches/ops/utils.rs
+++ b/benches/ops/utils.rs
@@ -16,9 +16,9 @@
use bytes::Bytes;
use once_cell::sync::Lazy;
+use opendal::services::*;
use opendal::Accessor;
use opendal::Operator;
-use opendal_test::services::*;
use rand::prelude::*;
pub static TOKIO: Lazy<tokio::runtime::Runtime> =
@@ -27,9 +27,9 @@
pub fn services() -> Vec<(&'static str, Option<Arc<dyn Accessor>>)> {
TOKIO.block_on(async {
vec![
- ("fs", fs::new().await.expect("init fs")),
- ("s3", s3::new().await.expect("init s3")),
- ("memory", memory::new().await.expect("init memory")),
+ ("fs", fs::tests::new().await.expect("init fs")),
+ ("s3", s3::tests::new().await.expect("init s3")),
+ ("memory", memory::tests::new().await.expect("init memory")),
]
})
}
diff --git a/opendal_test/Cargo.toml b/opendal_test/Cargo.toml
deleted file mode 100644
index 5233e11..0000000
--- a/opendal_test/Cargo.toml
+++ /dev/null
@@ -1,19 +0,0 @@
-[package]
-authors = ["Databend Authors <opensource@datafuselabs.com>"]
-categories = ["filesystem"]
-description = "Open Data Access Layer that connect the whole world together."
-edition = "2021"
-keywords = ["storage", "data", "s3"]
-license = "Apache-2.0"
-name = "opendal_test"
-publish = false
-repository = "https://github.com/datafuselabs/opendal"
-version = "0.0.0"
-
-[features]
-services-hdfs = ["opendal/services-hdfs"]
-
-[dependencies]
-dotenv = "0.15.0"
-opendal = { path = ".." }
-uuid = { version = "1.0.0", features = ["serde", "v4"] }
diff --git a/opendal_test/src/lib.rs b/opendal_test/src/lib.rs
deleted file mode 100644
index 01a22a8..0000000
--- a/opendal_test/src/lib.rs
+++ /dev/null
@@ -1,14 +0,0 @@
-// Copyright 2022 Datafuse Labs.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-pub mod services;
diff --git a/opendal_test/src/services/mod.rs b/opendal_test/src/services/mod.rs
deleted file mode 100644
index 8c88402..0000000
--- a/opendal_test/src/services/mod.rs
+++ /dev/null
@@ -1,19 +0,0 @@
-// Copyright 2022 Datafuse Labs.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-pub mod azblob;
-pub mod fs;
-#[cfg(feature = "services-hdfs")]
-pub mod hdfs;
-pub mod memory;
-pub mod s3;
diff --git a/src/services/azblob/mod.rs b/src/services/azblob/mod.rs
index d92e467..53b01a1 100644
--- a/src/services/azblob/mod.rs
+++ b/src/services/azblob/mod.rs
@@ -77,3 +77,7 @@
pub use backend::Builder;
mod object_stream;
+
+#[doc(hidden)]
+#[cfg(feature = "testing")]
+pub mod tests;
diff --git a/opendal_test/src/services/azblob.rs b/src/services/azblob/tests.rs
similarity index 92%
rename from opendal_test/src/services/azblob.rs
rename to src/services/azblob/tests.rs
index 6273fcb..7770530 100644
--- a/opendal_test/src/services/azblob.rs
+++ b/src/services/azblob/tests.rs
@@ -15,8 +15,7 @@
use std::io::Result;
use std::sync::Arc;
-use opendal::services::azblob;
-use opendal::Accessor;
+use crate::Accessor;
/// In order to test azblob service, please set the following environment variables:
/// - `OPENDAL_AZBLOB_TEST=on`: set to `on` to enable the test.
@@ -26,8 +25,6 @@
/// - `OPENDAL_AZBLOB_ACCOUNT_NAME=<account_name>`: set the account_name.
/// - `OPENDAL_AZBLOB_ACCOUNT_KEY=<account_key>`: set the account_key.
pub async fn new() -> Result<Option<Arc<dyn Accessor>>> {
- dotenv::from_filename(".env").ok();
-
if env::var("OPENDAL_AZBLOB_TEST").is_err() || env::var("OPENDAL_AZBLOB_TEST").unwrap() != "on"
{
return Ok(None);
@@ -36,7 +33,7 @@
let root = &env::var("OPENDAL_AZBLOB_ROOT").unwrap_or_else(|_| "/".to_string());
let root = format!("/{}/{}", root, uuid::Uuid::new_v4());
- let mut builder = azblob::Backend::build();
+ let mut builder = super::Backend::build();
builder
.root(&root)
diff --git a/src/services/fs/mod.rs b/src/services/fs/mod.rs
index 4af0b9b..50f95b2 100644
--- a/src/services/fs/mod.rs
+++ b/src/services/fs/mod.rs
@@ -53,3 +53,7 @@
mod error;
mod object_stream;
+
+#[doc(hidden)]
+#[cfg(feature = "testing")]
+pub mod tests;
diff --git a/opendal_test/src/services/fs.rs b/src/services/fs/tests.rs
similarity index 91%
rename from opendal_test/src/services/fs.rs
rename to src/services/fs/tests.rs
index 0778752..abb9a50 100644
--- a/opendal_test/src/services/fs.rs
+++ b/src/services/fs/tests.rs
@@ -16,16 +16,13 @@
use std::path::PathBuf;
use std::sync::Arc;
-use opendal::services::fs;
-use opendal::Accessor;
+use crate::Accessor;
/// In order to test fs service, please set the following environment variables:
///
/// - `OPENDAL_FS_TEST=on`: set to `on` to enable the test.
/// - `OPENDAL_FS_ROOT=<path>`: set the root directory of the test.
pub async fn new() -> Result<Option<Arc<dyn Accessor>>> {
- dotenv::from_filename(".env").ok();
-
if env::var("OPENDAL_FS_TEST").is_err() || env::var("OPENDAL_FS_TEST").unwrap() != "on" {
return Ok(None);
}
@@ -36,7 +33,7 @@
let root = PathBuf::from(root).join(uuid::Uuid::new_v4().to_string());
Ok(Some(
- fs::Backend::build()
+ super::Backend::build()
.root(root.to_str().unwrap())
.finish()
.await?,
diff --git a/src/services/hdfs/mod.rs b/src/services/hdfs/mod.rs
index a95c458..449ea38 100644
--- a/src/services/hdfs/mod.rs
+++ b/src/services/hdfs/mod.rs
@@ -92,3 +92,7 @@
mod error;
mod object_stream;
+
+#[doc(hidden)]
+#[cfg(feature = "testing")]
+pub mod tests;
diff --git a/opendal_test/src/services/hdfs.rs b/src/services/hdfs/tests.rs
similarity index 90%
rename from opendal_test/src/services/hdfs.rs
rename to src/services/hdfs/tests.rs
index 4a9014c..704948e 100644
--- a/opendal_test/src/services/hdfs.rs
+++ b/src/services/hdfs/tests.rs
@@ -15,8 +15,7 @@
use std::io::Result;
use std::sync::Arc;
-use opendal::services::hdfs;
-use opendal::Accessor;
+use crate::Accessor;
/// In order to test s3 service, please set the following environment variables:
///
@@ -24,8 +23,6 @@
/// - `OPENDAL_HDFS_ROOT=/path/to/dir`: set the root dir.
/// - `OPENDAL_HDFS_NAME_NODE=<name_node>`: set the name_node of the hdfs service.
pub async fn new() -> Result<Option<Arc<dyn Accessor>>> {
- dotenv::from_filename(".env").ok();
-
if env::var("OPENDAL_HDFS_TEST").is_err() || env::var("OPENDAL_HDFS_TEST").unwrap() != "on" {
return Ok(None);
}
@@ -33,7 +30,7 @@
let root = &env::var("OPENDAL_HDFS_ROOT").unwrap_or_else(|_| "/".to_string());
let root = format!("{}{}/", root, uuid::Uuid::new_v4());
- let mut builder = hdfs::Backend::build();
+ let mut builder = super::Backend::build();
builder.root(&root);
builder.name_node(
&env::var("OPENDAL_HDFS_NAME_NODE").expect("OPENDAL_HDFS_NAME_NODE must be set"),
diff --git a/src/services/memory/mod.rs b/src/services/memory/mod.rs
index d362019..41846a6 100644
--- a/src/services/memory/mod.rs
+++ b/src/services/memory/mod.rs
@@ -17,3 +17,7 @@
mod backend;
pub use backend::Backend;
pub use backend::Builder;
+
+#[doc(hidden)]
+#[cfg(feature = "testing")]
+pub mod tests;
diff --git a/opendal_test/src/services/memory.rs b/src/services/memory/tests.rs
similarity index 86%
rename from opendal_test/src/services/memory.rs
rename to src/services/memory/tests.rs
index 0cbae9a..cac8843 100644
--- a/opendal_test/src/services/memory.rs
+++ b/src/services/memory/tests.rs
@@ -15,19 +15,16 @@
use std::io::Result;
use std::sync::Arc;
-use opendal::services::memory;
-use opendal::Accessor;
+use crate::Accessor;
/// In order to test memory service, please set the following environment variables:
///
/// - `OPENDAL_MEMORY_TEST=on`: set to `on` to enable the test.
pub async fn new() -> Result<Option<Arc<dyn Accessor>>> {
- dotenv::from_filename(".env").ok();
-
if env::var("OPENDAL_MEMORY_TEST").is_err() || env::var("OPENDAL_MEMORY_TEST").unwrap() != "on"
{
return Ok(None);
}
- Ok(Some(memory::Backend::build().finish().await?))
+ Ok(Some(super::Backend::build().finish().await?))
}
diff --git a/src/services/s3/mod.rs b/src/services/s3/mod.rs
index 0867e78..05e9ebc 100644
--- a/src/services/s3/mod.rs
+++ b/src/services/s3/mod.rs
@@ -71,3 +71,7 @@
pub use backend::Builder;
mod object_stream;
+
+#[doc(hidden)]
+#[cfg(feature = "testing")]
+pub mod tests;
diff --git a/opendal_test/src/services/s3.rs b/src/services/s3/tests.rs
similarity index 94%
rename from opendal_test/src/services/s3.rs
rename to src/services/s3/tests.rs
index a06a50b..974995e 100644
--- a/opendal_test/src/services/s3.rs
+++ b/src/services/s3/tests.rs
@@ -15,8 +15,7 @@
use std::io::Result;
use std::sync::Arc;
-use opendal::services::s3;
-use opendal::Accessor;
+use crate::Accessor;
/// In order to test s3 service, please set the following environment variables:
///
@@ -27,8 +26,6 @@
/// - `OPENDAL_S3_ACCESS_KEY_ID=<access_key_id>`: set the access key id.
/// - `OPENDAL_S3_SECRET_ACCESS_KEY=<secret_access_key>`: set the secret access key.
pub async fn new() -> Result<Option<Arc<dyn Accessor>>> {
- dotenv::from_filename(".env").ok();
-
if env::var("OPENDAL_S3_TEST").is_err() || env::var("OPENDAL_S3_TEST").unwrap() != "on" {
return Ok(None);
}
@@ -36,7 +33,7 @@
let root = &env::var("OPENDAL_S3_ROOT").unwrap_or_else(|_| "/".to_string());
let root = format!("/{}/{}", root, uuid::Uuid::new_v4());
- let mut builder = s3::Backend::build();
+ let mut builder = super::Backend::build();
builder.root(&root);
builder.bucket(&env::var("OPENDAL_S3_BUCKET").expect("OPENDAL_S3_BUCKET must set"));
builder.endpoint(&env::var("OPENDAL_S3_ENDPOINT").unwrap_or_default());
diff --git a/tests/behavior/README.md b/tests/behavior/README.md
index 435bbbc..0b6bad2 100644
--- a/tests/behavior/README.md
+++ b/tests/behavior/README.md
@@ -29,11 +29,13 @@
Test all available backend.
```shell
-cargo test
+cargo test --features testing
```
Test specific backend.
```shell
-cargo test fs
+cargo test fs --features testing
```
+
+Notice: feature `testing` is required as we hide testing related utils under this feature.
diff --git a/tests/behavior/behavior.rs b/tests/behavior/behavior.rs
index 0c32f64..236d824 100644
--- a/tests/behavior/behavior.rs
+++ b/tests/behavior/behavior.rs
@@ -25,9 +25,9 @@
use anyhow::Result;
use futures::StreamExt;
use log::debug;
+use opendal::services;
use opendal::ObjectMode;
use opendal::Operator;
-use opendal_test::services;
use sha2::Digest;
use sha2::Sha256;
@@ -90,8 +90,9 @@
)*
async fn [< $test >]() -> Result<()> {
init_logger();
+ let _ = dotenv::dotenv();
- let acc = super::services::$service::new().await?;
+ let acc = super::services::$service::tests::new().await?;
if acc.is_none() {
return Ok(())
}
@@ -104,8 +105,12 @@
}
behavior_tests!(azblob, fs, memory, s3);
-#[cfg(feature = "services-hdfs")]
-behavior_tests!(hdfs);
+
+cfg_if::cfg_if! {
+ if #[cfg(feature = "services-hdfs")] {
+ behavior_tests!(hdfs);
+ }
+}
/// Create file with file path should succeed.
async fn test_create_file(op: Operator) -> Result<()> {
diff --git a/tests/behavior/main.rs b/tests/behavior/main.rs
index 368e04d..b2c1edc 100644
--- a/tests/behavior/main.rs
+++ b/tests/behavior/main.rs
@@ -13,7 +13,9 @@
// limitations under the License.
// Behavior test suites.
+#[cfg(feature = "testing")]
mod behavior;
+#[cfg(feature = "testing")]
mod utils;
pub fn init_logger() {
