feat(integrations/cloudfilter): introduce behavior tests (#4973)

* feat: init behavior tests

* docs

* feat: setup ci

* fix

* fix

* fix

* chore: bump `cloud-filter` to `0.0.5`

* fix

* fix: use `Get-Content` instead of `Get-FileHash` for checksum

* refactor: run behavior tests directly

* fix: planner

* fix

* ci: rework

* fix

* fix

.github/workflows/test_behavior_integration_cloudfilter.yml

@@ -0,0 +1,49 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Behavior Test Integration CloudFilter
+
+on:
+  push:
+    paths:
+      - "integrations/cloudfilter/**.rs"
+      - "integrations/cloudfilter/Cargo.toml"
+      - ".github/workflows/test_behavior_integration_cloudfilter.yml"
+  pull_request:
+    paths:
+      - "integrations/cloudfilter/**.rs"
+      - "integrations/cloudfilter/Cargo.toml"
+      - ".github/workflows/test_behavior_integration_cloudfilter.yml"
+
+jobs:
+  test:
+    name: fs / fixture_data
+    runs-on: windows-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Rust toolchain
+        uses: ./.github/actions/setup
+
+      - name: Test Integration CloudFilter
+        working-directory: integrations/cloudfilter
+        run: cargo test --test behavior
+        env:
+          OPENDAL_TEST: fs
+          OPENDAL_FS_ROOT: ../../fixtures/data
+          OPENDAL_DISABLE_RANDOM_ROOT: 'true'

integrations/cloudfilter/Cargo.toml

@@ -29,7 +29,7 @@ version = "0.0.0"
 [dependencies]
 anyhow = "1.0.86"
 bincode = "1.3.3"
-cloud-filter = "0.0.3"
+cloud-filter = "0.0.5"
 futures = "0.3.30"
 log = "0.4.17"
 opendal = { version = "0.49.0", path = "../../core" }
@@ -39,9 +39,17 @@ serde = { version = "1.0.203", features = ["derive"] }
 env_logger = "0.11.2"
 opendal = { version = "0.49.0", path = "../../core", features = [
   "services-fs",
+  "tests",
 ] }
 tokio = { version = "1.38.0", features = [
   "macros",
   "rt-multi-thread",
   "signal",
 ] }
+libtest-mimic = "0.7.3"
+powershell_script = "1.1.0"
+
+[[test]]
+harness = false
+name = "behavior"
+path = "tests/behavior/main.rs"

integrations/cloudfilter/src/lib.rs

@@ -25,11 +25,10 @@ use std::{
 
 use cloud_filter::{
     error::{CResult, CloudErrorKind},
-    filter::{info, ticket, Filter},
+    filter::{info, ticket, Filter, Request},
     metadata::Metadata,
     placeholder::{ConvertOptions, Placeholder},
     placeholder_file::PlaceholderFile,
-    request::Request,
     utility::{FileTime, WriteAt},
 };
 use file::FileBlob;

integrations/cloudfilter/tests/behavior/README.md

@@ -0,0 +1,12 @@
+# Behavior tests for OpenDAL™ Cloud Filter Integration
+
+Behavior tests are used to make sure every service works correctly.
+
+`cloudfilter_opendal` is readonly currently, so we assume `fixtures/data` is the root of the test data.
+
+## Run
+
+```pwsh
+cd .\integrations\cloudfilter
+$env:OPENDAL_TEST='fs'; $env:OPENDAL_FS_ROOT='../../fixtures/data'; $env:OPENDAL_DISABLE_RANDOM_ROOT='true'; cargo test --test behavior
+```

integrations/cloudfilter/tests/behavior/fetch_data.rs

@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use libtest_mimic::Failed;
+
+use crate::{
+    utils::{file_content, file_length},
+    ROOT_PATH,
+};
+
+pub fn test_fetch_data() -> Result<(), Failed> {
+    let files = [
+        (
+            "normal_file.txt",
+            include_str!("..\\..\\..\\..\\fixtures/data/normal_file.txt"),
+        ),
+        (
+            "special_file  !@#$%^&()_+-=;',.txt",
+            include_str!("..\\..\\..\\..\\fixtures/data/special_file  !@#$%^&()_+-=;',.txt"),
+        ),
+    ];
+    for (file, expected_content) in files {
+        let path = format!("{ROOT_PATH}\\{file}");
+        assert_eq!(
+            expected_content.len(),
+            file_length(&path).expect("file length"),
+            "file length",
+        );
+
+        assert_eq!(
+            expected_content,
+            file_content(&path).expect("file hash"),
+            "file hash",
+        )
+    }
+    Ok(())
+}

integrations/cloudfilter/tests/behavior/fetch_placeholder.rs

@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use libtest_mimic::Failed;
+
+use crate::{utils::list, ROOT_PATH};
+
+pub fn test_fetch_placeholder() -> Result<(), Failed> {
+    let files = ["normal_file.txt", "special_file  !@#$%^&()_+-=;',.txt"];
+    let dirs = ["normal_dir", "special_dir  !@#$%^&()_+-=;',"];
+
+    assert_eq!(
+        list(ROOT_PATH, "File").expect("list files"),
+        files,
+        "list files"
+    );
+    assert_eq!(
+        list(ROOT_PATH, "Directory").expect("list dirs"),
+        dirs,
+        "list dirs"
+    );
+
+    Ok(())
+}

integrations/cloudfilter/tests/behavior/main.rs

@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+mod fetch_data;
+mod fetch_placeholder;
+mod utils;
+
+use std::{
+    fs,
+    future::Future,
+    path::{Path, PathBuf},
+    pin::Pin,
+    process::ExitCode,
+};
+
+use cloud_filter::{
+    filter::AsyncBridge,
+    root::{
+        Connection, HydrationType, PopulationType, SecurityId, Session, SyncRootId,
+        SyncRootIdBuilder, SyncRootInfo,
+    },
+};
+use cloudfilter_opendal::CloudFilter;
+use libtest_mimic::{Arguments, Trial};
+use opendal::{raw::tests, Operator};
+use tokio::runtime::Handle;
+
+const PROVIDER_NAME: &str = "ro-cloudfilter";
+const DISPLAY_NAME: &str = "Test Cloud Filter";
+const ROOT_PATH: &str = "C:\\sync_root";
+
+#[tokio::main]
+async fn main() -> ExitCode {
+    let args = Arguments::from_args();
+
+    env_logger::init();
+
+    let Ok(Some(op)) = tests::init_test_service() else {
+        return ExitCode::SUCCESS;
+    };
+
+    if !Path::new(ROOT_PATH).try_exists().expect("try exists") {
+        fs::create_dir(ROOT_PATH).expect("create root dir");
+    }
+
+    let (sync_root_id, connection) = init(op);
+
+    let tests = vec![
+        Trial::test("fetch_data", fetch_data::test_fetch_data),
+        Trial::test(
+            "fetch_placeholder",
+            fetch_placeholder::test_fetch_placeholder,
+        ),
+    ];
+
+    let conclusion = libtest_mimic::run(&args, tests);
+
+    drop(connection);
+    sync_root_id.unregister().unwrap();
+    fs::remove_dir_all(ROOT_PATH).expect("remove root dir");
+
+    conclusion.exit_code()
+}
+
+fn init(
+    op: Operator,
+) -> (
+    SyncRootId,
+    Connection<AsyncBridge<CloudFilter, impl Fn(Pin<Box<dyn Future<Output = ()>>>)>>,
+) {
+    let sync_root_id = SyncRootIdBuilder::new(PROVIDER_NAME)
+        .user_security_id(SecurityId::current_user().unwrap())
+        .build();
+
+    if !sync_root_id.is_registered().unwrap() {
+        sync_root_id
+            .register(
+                SyncRootInfo::default()
+                    .with_display_name(DISPLAY_NAME)
+                    .with_hydration_type(HydrationType::Full)
+                    .with_population_type(PopulationType::Full)
+                    .with_icon("%SystemRoot%\\system32\\charmap.exe,0")
+                    .with_version("1.0.0")
+                    .with_recycle_bin_uri("http://cloudmirror.example.com/recyclebin")
+                    .unwrap()
+                    .with_path(ROOT_PATH)
+                    .unwrap(),
+            )
+            .unwrap();
+    }
+
+    let handle = Handle::current();
+    let connection = Session::new()
+        .connect_async(
+            ROOT_PATH,
+            CloudFilter::new(op, PathBuf::from(&ROOT_PATH)),
+            move |f| handle.clone().block_on(f),
+        )
+        .expect("create session");
+
+    (sync_root_id, connection)
+}

integrations/cloudfilter/tests/behavior/utils.rs

@@ -0,0 +1,51 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+use std::fmt::Display;
+
+use anyhow::Context;
+
+pub fn file_length(path: impl Display) -> anyhow::Result<usize> {
+    let len = powershell_script::run(&format!("(Get-Item \"{path}\" -Force).Length"))
+        .context("run powershell")?
+        .stdout()
+        .unwrap_or_default()
+        .trim()
+        .parse()
+        .context("parse length")?;
+
+    Ok(len)
+}
+
+pub fn file_content(path: impl Display) -> anyhow::Result<String> {
+    let content = powershell_script::run(&format!("Get-Content \"{path}\""))
+        .context("run powershell")?
+        .stdout()
+        .unwrap_or_default();
+    Ok(content)
+}
+
+pub fn list(path: impl Display, option: impl Display) -> anyhow::Result<Vec<String>> {
+    let entries = powershell_script::run(&format!("(Get-ChildItem \"{path}\" -{option}).Name"))
+        .context("run powershell")?
+        .stdout()
+        .unwrap_or_default()
+        .lines()
+        .map(Into::into)
+        .collect();
+    Ok(entries)
+}