diff --git a/ignite/metrics/clustering/calinski_harabasz_score.py b/ignite/metrics/clustering/calinski_harabasz_score.py
index fe58ac46151..79f8dc99ba5 100644
--- a/ignite/metrics/clustering/calinski_harabasz_score.py
+++ b/ignite/metrics/clustering/calinski_harabasz_score.py
@@ -11,8 +11,8 @@
 def _calinski_harabasz_score(features: Tensor, labels: Tensor) -> float:
     from sklearn.metrics import calinski_harabasz_score
 
-    np_features = features.numpy()
-    np_labels = labels.numpy()
+    np_features = features.cpu().numpy()
+    np_labels = labels.cpu().numpy()
     score = calinski_harabasz_score(np_features, np_labels)
     return score
 
diff --git a/ignite/metrics/clustering/davies_bouldin_score.py b/ignite/metrics/clustering/davies_bouldin_score.py
index b34ec69f51a..afea0518951 100644
--- a/ignite/metrics/clustering/davies_bouldin_score.py
+++ b/ignite/metrics/clustering/davies_bouldin_score.py
@@ -11,8 +11,8 @@
 def _davies_bouldin_score(features: Tensor, labels: Tensor) -> float:
     from sklearn.metrics import davies_bouldin_score
 
-    np_features = features.numpy()
-    np_labels = labels.numpy()
+    np_features = features.cpu().numpy()
+    np_labels = labels.cpu().numpy()
     score = davies_bouldin_score(np_features, np_labels)
     return score
 
diff --git a/ignite/metrics/clustering/silhouette_score.py b/ignite/metrics/clustering/silhouette_score.py
index 39b28c5d040..48a59d583ec 100644
--- a/ignite/metrics/clustering/silhouette_score.py
+++ b/ignite/metrics/clustering/silhouette_score.py
@@ -111,7 +111,7 @@ def __init__(
     def _silhouette_score(self, features: Tensor, labels: Tensor) -> float:
         from sklearn.metrics import silhouette_score
 
-        np_features = features.numpy()
-        np_labels = labels.numpy()
+        np_features = features.cpu().numpy()
+        np_labels = labels.cpu().numpy()
         score = silhouette_score(np_features, np_labels, **self._silhouette_kwargs)
         return score
diff --git a/ignite/metrics/regression/kendall_correlation.py b/ignite/metrics/regression/kendall_correlation.py
index 7ad87b22402..34d876a3659 100644
--- a/ignite/metrics/regression/kendall_correlation.py
+++ b/ignite/metrics/regression/kendall_correlation.py
@@ -16,8 +16,8 @@ def _get_kendall_tau(variant: str = "b") -> Callable[[Tensor, Tensor], float]:
         raise ValueError(f"variant accepts 'b' or 'c', got {variant!r}.")
 
     def _tau(predictions: Tensor, targets: Tensor) -> float:
-        np_preds = predictions.flatten().numpy()
-        np_targets = targets.flatten().numpy()
+        np_preds = predictions.flatten().cpu().numpy()
+        np_targets = targets.flatten().cpu().numpy()
         r = kendalltau(np_preds, np_targets, variant=variant).statistic
         return r
 
diff --git a/ignite/metrics/regression/spearman_correlation.py b/ignite/metrics/regression/spearman_correlation.py
index 7f126d6e56b..cbd89f67c9d 100644
--- a/ignite/metrics/regression/spearman_correlation.py
+++ b/ignite/metrics/regression/spearman_correlation.py
@@ -12,8 +12,8 @@
 def _spearman_r(predictions: Tensor, targets: Tensor) -> float:
     from scipy.stats import spearmanr
 
-    np_preds = predictions.flatten().numpy()
-    np_targets = targets.flatten().numpy()
+    np_preds = predictions.flatten().cpu().numpy()
+    np_targets = targets.flatten().cpu().numpy()
     r = spearmanr(np_preds, np_targets).statistic
     return r
 
diff --git a/tests/common_test_functionality.sh b/tests/common_test_functionality.sh
index 6e60947f927..91003eddc09 100644
--- a/tests/common_test_functionality.sh
+++ b/tests/common_test_functionality.sh
@@ -85,7 +85,6 @@ run_tests() {
         skip_distrib_opt=""
     fi
 
-
     echo [pytest] > pytest.ini ; echo "cache_dir=${cache_dir}" >> pytest.ini
 
     # Assemble options for the pytest command
@@ -103,8 +102,8 @@ run_tests() {
 
     # Run the command
     if [ "$trap_deselected_exit_code" -eq "1" ]; then
-        CUDA_VISIBLE_DEVICES="" eval "pytest ${pytest_args}" || { exit_code=$?; if [ "$exit_code" -eq ${last_failed_no_failures_code} ]; then echo "All tests deselected"; else exit $exit_code; fi; }
+        eval "pytest ${pytest_args}" || { exit_code=$?; if [ "$exit_code" -eq ${last_failed_no_failures_code} ]; then echo "All tests deselected"; else exit $exit_code; fi; }
     else
-        CUDA_VISIBLE_DEVICES="" eval "pytest ${pytest_args}"
+        eval "pytest ${pytest_args}"
     fi
 }
diff --git a/tests/ignite/metrics/test_hsic.py b/tests/ignite/metrics/test_hsic.py
index 57af5fa2862..28fe5c1f97d 100644
--- a/tests/ignite/metrics/test_hsic.py
+++ b/tests/ignite/metrics/test_hsic.py
@@ -139,10 +139,10 @@ def test_integration(self, sigma_x: float, sigma_y: float):
             metric_devices.append(device)
 
         for metric_device in metric_devices:
-            x = torch.randn((n_iters * batch_size, n_dims_x)).float().to(device)
+            x = torch.randn((n_iters * batch_size, n_dims_x), device=device).float()
 
             lin = nn.Linear(n_dims_x, n_dims_y).to(device)
-            y = torch.sin(lin(x) * 100) + torch.randn(n_iters * batch_size, n_dims_y) * 1e-4
+            y = torch.sin(lin(x) * 100) + torch.randn(n_iters * batch_size, n_dims_y, device=x.device) * 1e-4
 
             def data_loader(i, input_x, input_y):
                 return input_x[i * batch_size : (i + 1) * batch_size], input_y[i * batch_size : (i + 1) * batch_size]
diff --git a/tests/run_cpu_tests.sh b/tests/run_cpu_tests.sh
index 8d387f5542e..f52988a6818 100644
--- a/tests/run_cpu_tests.sh
+++ b/tests/run_cpu_tests.sh
@@ -6,8 +6,7 @@ skip_distrib_tests=${SKIP_DISTRIB_TESTS:-0}
 use_last_failed=${USE_LAST_FAILED:-0}
 match_tests_expression=${1:-""}
 
-
-run_tests \
+CUDA_VISIBLE_DEVICES="" run_tests \
     --core_args "--tx 4*popen//python=python -vvv tests/ignite" \
     --cache_dir ".cpu-not-distrib" \
     --skip_distrib_tests "${skip_distrib_tests}" \
@@ -21,7 +20,7 @@ if [ "${skip_distrib_tests}" -eq "1" ]; then
 fi
 
 # Run 2 processes with --dist=each
-run_tests \
+CUDA_VISIBLE_DEVICES="" run_tests \
     --core_args "-m distributed -vvv tests/ignite" \
     --world_size 2 \
     --cache_dir ".cpu-distrib" \
diff --git a/tests/run_gpu_tests.sh b/tests/run_gpu_tests.sh
index 26497f19c83..c86d1d0746e 100644
--- a/tests/run_gpu_tests.sh
+++ b/tests/run_gpu_tests.sh
@@ -2,26 +2,26 @@
 source "$(dirname "$0")/common_test_functionality.sh"
 set -xeu
 
-skip_distrib_tests=${SKIP_DISTRIB_TESTS:-1}
+# https://pubs.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_06_02
+skip_distrib_tests=${SKIP_DISTRIB_TESTS:-0}
 use_last_failed=${USE_LAST_FAILED:-0}
 ngpus=${1:-1}
 
 match_tests_expression=${2:-""}
 if [ -z "$match_tests_expression" ]; then
-    cuda_pattern="cuda"
+    cuda_pattern="cuda or nccl or gloo"
 else
-    cuda_pattern="cuda and $match_tests_expression"
+    cuda_pattern="(cuda or nccl or gloo) and $match_tests_expression"
 fi
 
 run_tests \
-    --core_args "-vvv tests/ignite" \
+    --core_args "-vvv tests/ignite -m 'not distributed'" \
     --cache_dir ".gpu-cuda" \
     --skip_distrib_tests "${skip_distrib_tests}" \
     --use_coverage 1 \
     --match_tests_expression "${cuda_pattern}" \
     --use_last_failed ${use_last_failed}
 
-# https://pubs.opengroup.org/onlinepubs/009695399/utilities/xcu_chap02.html#tag_02_06_02
 if [ "${skip_distrib_tests}" -eq "1" ]; then
     exit 0
 fi