diff --git a/conftest.py b/conftest.py
index a8b12e99..1f985c3b 100644
--- a/conftest.py
+++ b/conftest.py
@@ -191,21 +191,27 @@ def hosts(pytestconfig):
     hosts_args = pytestconfig.getoption("hosts")
     hosts_split = [hostlist.split(',') for hostlist in hosts_args]
     hostname_list = list(itertools.chain(*hosts_split))
-    try:
-        host_list = []
-        for hostname_or_ip in hostname_list:
-            host_list.append(setup_host(hostname_or_ip, config=pytestconfig))
-
-        if not host_list:
-            pytest.fail("This test requires at least one --hosts parameter")
-        yield host_list
 
-    finally:
+    host_list = []
+    def _cleanup_hosts():
         for host in host_list:
             if host.nested:
                 logging.info("Destroying nested host's VM %s", host.nested.uuid)
                 host.nested.destroy(verify=True)
 
+    try:
+        for hostname_or_ip in hostname_list:
+            host_list.append(setup_host(hostname_or_ip, config=pytestconfig))
+    except Exception:
+        _cleanup_hosts()
+        raise
+
+    if not host_list:
+        pytest.fail("This test requires at least one --hosts parameter")
+    yield host_list
+
+    _cleanup_hosts
+
 @pytest.fixture(scope='session')
 def registered_xo_cli():
     # The fixture is not responsible for establishing the connection.