From 4d3e1f2944387005fa3a5a8e81926b57e098bf3a Mon Sep 17 00:00:00 2001 From: Tiago Castro Date: Wed, 19 Jul 2023 00:29:53 +0100 Subject: [PATCH 1/2] test: fix python tests Now that the nexus stays shutdown on all children failure, dd might not complete with error. To address this, add smaller delay and ctrl loss timeout. Also add missing sudo for xfs_info. Signed-off-by: Tiago Castro --- test/python/common/nvme.py | 6 ++++-- test/python/tests/nexus/test_multi_nexus.py | 4 +++- test/python/tests/nexus/test_nexus.py | 2 +- test/python/tests/nexus_fault/test_nexus_fault.py | 6 +++--- test/python/v1/nexus/test_multi_nexus.py | 4 +++- test/python/v1/nexus/test_nexus.py | 2 +- 6 files changed, 15 insertions(+), 9 deletions(-) diff --git a/test/python/common/nvme.py b/test/python/common/nvme.py index 25176c88b..38e8dead8 100644 --- a/test/python/common/nvme.py +++ b/test/python/common/nvme.py @@ -56,13 +56,15 @@ async def nvme_remote_discover(remote, uri): raise ValueError("uri {} is not discovered".format(u.path[1:])) -def nvme_connect(uri): +def nvme_connect(uri, delay=10, tmo=600): u = urlparse(uri) port = u.port host = u.hostname nqn = u.path[1:] - command = "sudo nvme connect -t tcp -s {0} -a {1} -n {2}".format(port, host, nqn) + command = ( + f"sudo nvme connect -t tcp -s {port} -a {host} -n {nqn} -c {delay} -l {tmo}" + ) subprocess.run(command, check=True, shell=True, capture_output=False) time.sleep(1) command = "sudo nvme list -v -o json" diff --git a/test/python/tests/nexus/test_multi_nexus.py b/test/python/tests/nexus/test_multi_nexus.py index d82e0656e..b8ed3ef03 100644 --- a/test/python/tests/nexus/test_multi_nexus.py +++ b/test/python/tests/nexus/test_multi_nexus.py @@ -7,6 +7,7 @@ import pytest import asyncio import uuid as guid +import pytest_asyncio NEXUS_COUNT = 15 DESTROY_COUNT = 7 @@ -114,7 +115,7 @@ def connect_devices(create_nexuses): nvme_disconnect(nexus) -@pytest.fixture +@pytest_asyncio.fixture async def mount_devices(connect_devices): "Create and mount a filesystem on each nvmf connected device." for dev in connect_devices: @@ -126,6 +127,7 @@ async def mount_devices(connect_devices): for dev in connect_devices: await run_cmd_async(f"sudo umount /mnt{dev}") + await run_cmd_async(f"sudo rm -rf /mnt/dev") @pytest.mark.asyncio diff --git a/test/python/tests/nexus/test_nexus.py b/test/python/tests/nexus/test_nexus.py index 5f1af0ccb..607ed99e2 100644 --- a/test/python/tests/nexus/test_nexus.py +++ b/test/python/tests/nexus/test_nexus.py @@ -447,7 +447,7 @@ def test_nexus_preempt_key( # verify write error with nexus on ms3 uri = create_nexus_v2 - dev = nvme_connect(uri) + dev = nvme_connect(uri, 1, 1) job = "sudo dd if=/dev/urandom of={0} bs=512 count=1".format(dev) try: diff --git a/test/python/tests/nexus_fault/test_nexus_fault.py b/test/python/tests/nexus_fault/test_nexus_fault.py index c9ca1a1c5..a2402f613 100644 --- a/test/python/tests/nexus_fault/test_nexus_fault.py +++ b/test/python/tests/nexus_fault/test_nexus_fault.py @@ -122,9 +122,9 @@ def _(mounted_nexus): """the initiator filesystem should not be shutdown.""" try: # xfs_info should still be working as the fs is not shutdown - run_cmd(f"xfs_info {mounted_nexus}") - except: - pytest.fail(f"Filesystem on {mounted_nexus} should not be shutdown") + run_cmd(f"sudo xfs_info {mounted_nexus}") + except Exception as e: + pytest.fail(f"Filesystem on {mounted_nexus} should not be shutdown: {e}") @pytest.fixture(scope="module") diff --git a/test/python/v1/nexus/test_multi_nexus.py b/test/python/v1/nexus/test_multi_nexus.py index 8da11200a..a88d6b306 100644 --- a/test/python/v1/nexus/test_multi_nexus.py +++ b/test/python/v1/nexus/test_multi_nexus.py @@ -7,6 +7,7 @@ import pytest import asyncio import uuid as guid +import pytest_asyncio NEXUS_COUNT = 15 DESTROY_COUNT = 7 @@ -139,7 +140,7 @@ def connect_devices(create_nexuses): nvme_disconnect(nexus) -@pytest.fixture +@pytest_asyncio.fixture async def mount_devices(connect_devices): "Create and mount a filesystem on each nvmf connected device." for dev in connect_devices: @@ -151,6 +152,7 @@ async def mount_devices(connect_devices): for dev in connect_devices: await run_cmd_async(f"sudo umount /mnt{dev}") + await run_cmd_async(f"sudo rm -rf /mnt/dev") @pytest.mark.asyncio diff --git a/test/python/v1/nexus/test_nexus.py b/test/python/v1/nexus/test_nexus.py index ca978f858..1856fb455 100644 --- a/test/python/v1/nexus/test_nexus.py +++ b/test/python/v1/nexus/test_nexus.py @@ -435,7 +435,7 @@ def test_nexus_preempt_key( # verify write error with nexus on ms3 uri = create_nexus - dev = nvme_connect(uri) + dev = nvme_connect(uri, 1, 1) job = "sudo dd if=/dev/urandom of={0} bs=512 count=1".format(dev) try: From 614dcf1d2e3da53af4b15cc70e9f689d6b4793cb Mon Sep 17 00:00:00 2001 From: Tiago Castro Date: Wed, 19 Jul 2023 10:22:17 +0100 Subject: [PATCH 2/2] ci: cleanup test runs Signed-off-by: Tiago Castro --- Jenkinsfile | 4 ++-- scripts/cargo-test.sh | 18 +++++++----------- scripts/clean-cargo-tests.sh | 19 +++++++++++++++++++ scripts/grpc-test.sh | 11 +++++++++++ 4 files changed, 39 insertions(+), 13 deletions(-) create mode 100755 scripts/clean-cargo-tests.sh diff --git a/Jenkinsfile b/Jenkinsfile index 202ef9239..d969fc24c 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -155,8 +155,7 @@ pipeline { } post { always { - // in case of abnormal termination of any nvmf test - sh 'sudo nvme disconnect-all' + sh 'nix-shell --run "./scripts/clean-cargo-tests.sh" ci.nix' sh 'sudo ./scripts/check-coredumps.sh --since "${START_DATE}"' } } @@ -190,6 +189,7 @@ pipeline { } post { always { + sh 'nix-shell --run "./scripts/clean-cargo-tests.sh" ci.nix' junit '*-xunit-report.xml' sh 'sudo ./scripts/check-coredumps.sh --since "${START_DATE}"' } diff --git a/scripts/cargo-test.sh b/scripts/cargo-test.sh index bf5d5c1c0..ae0e870f8 100755 --- a/scripts/cargo-test.sh +++ b/scripts/cargo-test.sh @@ -1,24 +1,20 @@ #!/usr/bin/env bash -SCRIPTDIR=$(dirname "$0") +SCRIPTDIR="$(realpath "$(dirname "$0")")" cleanup_handler() { - for c in $(docker ps -a --filter "label=io.mayastor.test.name" --format '{{.ID}}') ; do - docker kill "$c" || true - docker rm "$c" || true - done - - for n in $(docker network ls --filter "label=io.mayastor.test.name" --format '{{.ID}}') ; do - docker network rm "$n" || true - done + ERROR=$? + "$SCRIPTDIR"/clean-cargo-tests.sh || true + if [ $ERROR != 0 ]; then exit $ERROR; fi } -trap cleanup_handler ERR INT QUIT TERM HUP - echo "running cargo-test..." echo "rustc version:" rustc --version +leanup_handler +trap cleanup_handler INT QUIT TERM HUP EXIT + set -euxo pipefail export PATH=$PATH:${HOME}/.cargo/bin ( cd jsonrpc && cargo test ) diff --git a/scripts/clean-cargo-tests.sh b/scripts/clean-cargo-tests.sh new file mode 100755 index 000000000..437080ebd --- /dev/null +++ b/scripts/clean-cargo-tests.sh @@ -0,0 +1,19 @@ +SCRIPT_DIR="$(dirname "$0")" +ROOT_DIR=$(realpath "$SCRIPT_DIR/..") + +sudo nvme disconnect-all + +for c in $(docker ps -a --filter "label=io.mayastor.test.name" --format '{{.ID}}') ; do + docker kill "$c" + docker rm "$c" +done + +for n in $(docker network ls --filter "label=io.mayastor.test.name" --format '{{.ID}}') ; do + docker network rm "$n" || ( sudo systemctl restart docker && docker network rm "$n" ) +done + +# Kill's processes running off the workspace cargo binary location +ps aux | grep "$ROOT_DIR/target" | grep -v -e sudo -e grep | awk '{ print $2 }' | xargs -I% sudo kill -9 % + +exit 0 + diff --git a/scripts/grpc-test.sh b/scripts/grpc-test.sh index de607b930..4c651b8af 100755 --- a/scripts/grpc-test.sh +++ b/scripts/grpc-test.sh @@ -1,5 +1,16 @@ #!/usr/bin/env bash +SCRIPTDIR="$(realpath "$(dirname "$0")")" + +cleanup_handler() { + ERROR=$? + "$SCRIPTDIR"/clean-cargo-tests.sh || true + if [ $ERROR != 0 ]; then exit $ERROR; fi +} + +cleanup_handler +trap cleanup_handler INT QUIT TERM HUP EXIT + set -euxo pipefail export PATH="$PATH:${HOME}/.cargo/bin"