diff --git a/io-engine-tests/src/lib.rs b/io-engine-tests/src/lib.rs index 175cb665e..09ccfa5b0 100644 --- a/io-engine-tests/src/lib.rs +++ b/io-engine-tests/src/lib.rs @@ -8,6 +8,7 @@ use std::{io, io::Write, process::Command, time::Duration}; use crossbeam::channel::{after, select, unbounded}; use once_cell::sync::OnceCell; +use regex::Regex; use run_script::{self, ScriptOptions}; use url::{ParseError, Url}; @@ -36,6 +37,8 @@ pub mod snapshot; pub mod test; pub mod test_task; +use cli_tools::run_command_args; + pub use compose::MayastorTest; /// call F cnt times, and sleep for a duration between each invocation @@ -550,6 +553,86 @@ pub fn reactor_run_millis(milliseconds: u64) { reactor_poll!(r); } +// We try to figure out the net interface that is being used to connect +// to internet(public IP). Then use that interface name to setup rdma rxe +// device. +// +// # host we want to "reach" +// host=google.com +// +// # get the ip of that host (works with dns and /etc/hosts. In case we get +// # multiple IP addresses, we just want one of them +// host_ip=$(getent ahosts "$host" | awk '{print $1; exit}') +// +// # only list the interface used to reach a specific host/IP. We only want the +// part # between dev and the remainder (use grep for that) +// ip route get "$host_ip" | grep -Po '(?<=(dev ))(\S+)' +pub fn setup_rdma_rxe_device() -> String { + let test_host = "google.com"; + let ns_entries = run_command_args( + "getent", + vec!["ahosts", test_host], + Some("get ip of test host"), + ) + .unwrap(); + + let test_host_ip = ns_entries + .1 + .first() + .unwrap() + .to_string_lossy() + .split_whitespace() + .next() + .unwrap() + .to_owned(); + + let ent = &run_command_args( + "ip", + vec!["route", "get", test_host_ip.as_str()], + Some("get routing entry"), + ) + .unwrap() + .1[0]; + + // match/grep the interface name + let pattern = r"dev (\S+)"; + let re = Regex::new(pattern).unwrap(); + + let iface = re + .captures(ent.to_str().unwrap()) + .unwrap() + .get(1) + .expect("interface not found") + .as_str(); + println!("Using interface {} to create rdma rxe device", iface); + + // now create the software rdma device. + let _ = run_command_args( + "rdma", + vec![ + "link", + "add", + "io-engine-rxe0", + "type", + "rxe", + "netdev", + iface, + ], + Some("Create rxe device"), + ); + + iface.to_string() +} + +pub fn delete_rdma_rxe_device() { + let _ = run_command_args( + "rdma", + vec!["link", "delete", "io-engine-rxe0"], + Some("Delete rxe device"), + ) + .expect("rxe device delete"); +} + pub fn composer_init() { std::fs::create_dir_all("/var/run/dpdk").ok(); let path = std::path::PathBuf::from(std::env!("CARGO_MANIFEST_DIR")); diff --git a/io-engine-tests/src/nvme.rs b/io-engine-tests/src/nvme.rs index 2701bf6e6..6faf5b4b0 100644 --- a/io-engine-tests/src/nvme.rs +++ b/io-engine-tests/src/nvme.rs @@ -15,7 +15,7 @@ pub struct NmveConnectGuard { impl NmveConnectGuard { pub fn connect(target_addr: &str, nqn: &str) -> Self { - nvme_connect(target_addr, nqn, true); + nvme_connect(target_addr, nqn, "tcp", true); Self { nqn: nqn.to_string(), @@ -85,11 +85,12 @@ pub fn nvme_discover(target_addr: &str) -> Vec> { pub fn nvme_connect( target_addr: &str, nqn: &str, + transport: &str, must_succeed: bool, ) -> ExitStatus { let status = Command::new("nvme") .args(["connect"]) - .args(["-t", "tcp"]) + .args(["-t", transport]) .args(["-a", target_addr]) .args(["-s", "8420"]) .args(["-c", "1"]) diff --git a/io-engine/tests/nexus_io.rs b/io-engine/tests/nexus_io.rs index f38d922b2..6c9e2a3d7 100644 --- a/io-engine/tests/nexus_io.rs +++ b/io-engine/tests/nexus_io.rs @@ -200,12 +200,12 @@ async fn nexus_io_multipath() { .unwrap(); let nqn = format!("{HOSTNQN}:nexus-{NEXUS_UUID}"); - nvme_connect("127.0.0.1", &nqn, true); + nvme_connect("127.0.0.1", &nqn, "tcp", true); // The first attempt will fail with "Duplicate cntlid x with y" error from // kernel for i in 0 .. 2 { - let status_c0 = nvme_connect(&ip0.to_string(), &nqn, false); + let status_c0 = nvme_connect(&ip0.to_string(), &nqn, "tcp", false); if i == 0 && status_c0.success() { break; } @@ -270,7 +270,7 @@ async fn nexus_io_multipath() { // Connect to remote replica to check key registered let rep_nqn = format!("{HOSTNQN}:{REPL_UUID}"); - nvme_connect(&ip0.to_string(), &rep_nqn, true); + nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true); let rep_dev = get_mayastor_nvme_device(); @@ -404,7 +404,7 @@ async fn nexus_io_resv_acquire() { // Connect to remote replica to check key registered let rep_nqn = format!("{HOSTNQN}:{REPL_UUID}"); - nvme_connect(&ip0.to_string(), &rep_nqn, true); + nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true); let rep_dev = get_mayastor_nvme_device(); @@ -601,7 +601,7 @@ async fn nexus_io_resv_preempt() { // Connect to remote replica to check key registered let rep_nqn = format!("{HOSTNQN}:{REPL_UUID}"); - nvme_connect(&ip0.to_string(), &rep_nqn, true); + nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true); let rep_dev = get_mayastor_nvme_device(); @@ -748,7 +748,7 @@ async fn nexus_io_resv_preempt() { .await .unwrap(); - nvme_connect(&ip0.to_string(), &rep_nqn, true); + nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true); let rep_dev = get_mayastor_nvme_device(); // After restart the reservations should still be in place! @@ -899,7 +899,7 @@ async fn nexus_io_resv_preempt_tabled() { // Connect to remote replica to check key registered let rep_nqn = format!("{HOSTNQN}:{REPL_UUID}"); - nvme_connect(&ip0.to_string(), &rep_nqn, true); + nvme_connect(&ip0.to_string(), &rep_nqn, "tcp", true); let rep_dev = get_mayastor_nvme_device(); diff --git a/io-engine/tests/nvmf.rs b/io-engine/tests/nvmf.rs index 37310cb2e..cb8de4635 100644 --- a/io-engine/tests/nvmf.rs +++ b/io-engine/tests/nvmf.rs @@ -1,5 +1,6 @@ use io_engine::{ bdev_api::bdev_create, + constants::NVME_NQN_PREFIX, core::{ mayastor_env_stop, MayastorCliArgs, @@ -10,21 +11,79 @@ use io_engine::{ subsys::{NvmfSubsystem, SubType}, }; +use io_engine_tests::{delete_rdma_rxe_device, setup_rdma_rxe_device}; + pub mod common; -use common::compose::{ - rpc::v0::{ - mayastor::{BdevShareRequest, BdevUri, CreateReply}, - GrpcConnect, +use common::{ + compose::{ + rpc::{ + v0::{ + mayastor::{ + BdevShareRequest, + BdevUri, + CreateReply, + ShareProtocolNexus, + }, + GrpcConnect, + }, + v1::{ + nexus::{CreateNexusRequest, PublishNexusRequest}, + pool::CreatePoolRequest, + replica::CreateReplicaRequest, + GrpcConnect as v1GrpcConnect, + RpcHandle, + }, + }, + Binary, + Builder, + ComposeTest, + NetworkMode, }, - Binary, - Builder, - ComposeTest, + nvme::{nvme_connect, nvme_disconnect_nqn}, }; use regex::Regex; static DISKNAME1: &str = "/tmp/disk1.img"; static BDEVNAME1: &str = "aio:///tmp/disk1.img?blk_size=512"; +fn nexus_uuid() -> String { + "cdc2a7db-3ac3-403a-af80-7fadc1581c47".to_string() +} +fn nexus_name() -> String { + "nexus0".to_string() +} +fn repl_uuid() -> String { + "65acdaac-14c4-41d8-a55e-d03bfd7185a4".to_string() +} +fn repl_name() -> String { + "repl0".to_string() +} +fn pool_uuid() -> String { + "6e3c062c-293b-46e6-8ab3-ff13c1643437".to_string() +} +fn pool_name() -> String { + "tpool".to_string() +} + +pub async fn create_nexus(h: &mut RpcHandle, children: Vec) { + h.nexus + .create_nexus(CreateNexusRequest { + name: nexus_name(), + uuid: nexus_uuid(), + size: 60 * 1024 * 1024, + min_cntl_id: 1, + max_cntl_id: 1, + resv_key: 1, + preempt_key: 0, + children, + nexus_info_key: nexus_name(), + resv_type: None, + preempt_policy: 0, + }) + .await + .unwrap(); +} + #[common::spdk_test] fn nvmf_target() { common::mayastor_test_init(); @@ -195,3 +254,90 @@ async fn nvmf_set_target_interface() { // test_fail("10.15.0.0/16", vec!["-T", "mac:123"]).await; // test_fail("10.15.0.0/16", vec!["-T", "ip:hello"]).await; } + +#[tokio::test] +#[ignore] +async fn test_rdma_target() { + common::composer_init(); + + let iface = setup_rdma_rxe_device(); + let test = Builder::new() + .name("cargo-test") + .network_mode(NetworkMode::Host) + .add_container_bin( + "ms_0", + Binary::from_dbg("io-engine") + .with_args(vec![ + "-l", + "1,2", + "--enable-rdma", + "-T", + iface.as_str(), + ]) + .with_privileged(Some(true)), + ) + .with_clean(true) + .build() + .await + .unwrap(); + + let conn = v1GrpcConnect::new(&test); + let mut hdl = conn.grpc_handle("ms_0").await.unwrap(); + println!("ms_0 grpc endpoint {:?}", hdl.endpoint); + + hdl.pool + .create_pool(CreatePoolRequest { + name: pool_name(), + uuid: Some(pool_uuid()), + pooltype: 0, + disks: vec!["malloc:///disk0?size_mb=100".into()], + cluster_size: None, + md_args: None, + }) + .await + .unwrap(); + + hdl.replica + .create_replica(CreateReplicaRequest { + name: repl_name(), + uuid: repl_uuid(), + pooluuid: pool_uuid(), + size: 80 * 1024 * 1024, + thin: false, + share: 1, + ..Default::default() + }) + .await + .unwrap(); + + let child0 = format!("bdev:///{}", repl_name()); + create_nexus(&mut hdl, vec![child0.clone()]).await; + let device_uri = hdl + .nexus + .publish_nexus(PublishNexusRequest { + uuid: nexus_uuid(), + key: "".to_string(), + share: ShareProtocolNexus::NexusNvmf as i32, + ..Default::default() + }) + .await + .expect("Failed to publish nexus") + .into_inner() + .nexus + .unwrap() + .device_uri; + + let url = url::Url::parse(device_uri.as_str()).unwrap(); + assert!(url.scheme() == "nvmf+rdma+tcp"); + + let host = url.host_str().unwrap(); + let nqn = format!("{NVME_NQN_PREFIX}:{}", nexus_name()); + let conn_status = nvme_connect(host, &nqn, "rdma", true); + assert!(conn_status.success()); + + nvme_disconnect_nqn(&nqn); + // Explicitly destroy this test's containers so that rxe device can be + // deleted. + test.down().await; + delete_rdma_rxe_device(); +} diff --git a/io-engine/tests/snapshot_nexus.rs b/io-engine/tests/snapshot_nexus.rs index 2b5b2726f..fc5a3471a 100755 --- a/io-engine/tests/snapshot_nexus.rs +++ b/io-engine/tests/snapshot_nexus.rs @@ -680,7 +680,7 @@ async fn test_snapshot_ancestor_usage() { * should now own all new data. */ let nqn = format!("{NVME_NQN_PREFIX}:{}", nexus_name()); - nvme_connect("127.0.0.1", &nqn, true); + nvme_connect("127.0.0.1", &nqn, "tcp", true); let (s, r) = oneshot::channel::<()>(); tokio::spawn(async move { diff --git a/scripts/cargo-test.sh b/scripts/cargo-test.sh index 9d5269cc8..168bfc6dc 100755 --- a/scripts/cargo-test.sh +++ b/scripts/cargo-test.sh @@ -19,6 +19,17 @@ trap cleanup_handler INT QUIT TERM HUP EXIT export PATH=$PATH:${HOME}/.cargo/bin set -euxo pipefail +# Warn if rdma-rxe and nvme-rdme kernel modules are not +# available. Absence of rdma-rxe can be ignored on hardware +# RDMA setups. +if ! lsmod | grep -q rdma_rxe; then + echo "Warning: rdma_rxe kernel module is not loaded. Please load it for rdma tests to work." +fi + +if ! lsmod | grep -q nvme_rdma; then + echo "Warning: nvme_rdma kernel module is not loaded. Please load it for rdma tests to work." +fi + ( cd jsonrpc && cargo test ) # test dependencies cargo build --bins --features=io-engine-testing diff --git a/scripts/clean-cargo-tests.sh b/scripts/clean-cargo-tests.sh index d259ace32..a71a1938a 100755 --- a/scripts/clean-cargo-tests.sh +++ b/scripts/clean-cargo-tests.sh @@ -21,6 +21,9 @@ done # Delete the directory too nix-sudo rmdir --ignore-fail-on-non-empty "/tmp/io-engine-tests" 2>/dev/null +# If there was a soft rdma device created and left undeleted by nvmf rdma test, +# delete that now. Not removing rdma-rxe kernel module. +nix-sudo rdma link delete io-engine-rxe0 2>/dev/null for c in $(docker ps -a --filter "label=io.composer.test.name" --format '{{.ID}}') ; do docker kill "$c"