[FIX] tqdm not displaying on notebooks (#878)

* test tqdm change

* replace the corrector

* replace tqdm

* fix zip error

* replace all with normal calls of tqdm

* run isort

* change list to dict

Author: jdkent

nimare/decode/continuous.py

@@ -19,13 +19,7 @@
 from nimare.meta.cbma.mkda import MKDAChi2
 from nimare.results import MetaResult
 from nimare.stats import pearson
-from nimare.utils import (
-    _check_ncores,
-    _check_type,
-    _safe_transform,
-    get_masker,
-    tqdm_joblib,
-)
+from nimare.utils import _check_ncores, _check_type, _safe_transform, get_masker
 
 LGR = logging.getLogger(__name__)
 
@@ -200,12 +194,15 @@ def _fit(self, dataset):
             MetaResult with meta-analytic maps and masker added.
         """
         n_features = len(self.features_)
-        with tqdm_joblib(tqdm(total=n_features)):
-            maps = dict(
-                Parallel(n_jobs=self.n_cores)(
+        maps = {
+            r: v
+            for r, v in tqdm(
+                Parallel(return_as="generator", n_jobs=self.n_cores)(
                     delayed(self._run_fit)(feature, dataset) for feature in self.features_
-                )
+                ),
+                total=n_features,
             )
+        }
 
         self.results_ = MetaResult(self, mask=dataset.masker, maps=maps)
 
@@ -390,12 +387,15 @@ def _fit(self, dataset):
             MetaResult with meta-analytic maps and masker added.
         """
         n_features = len(self.features_)
-        with tqdm_joblib(tqdm(total=n_features)):
-            maps = dict(
-                Parallel(n_jobs=self.n_cores)(
+        maps = {
+            r: v
+            for r, v in tqdm(
+                Parallel(return_as="generator", n_jobs=self.n_cores)(
                     delayed(self._run_fit)(feature, dataset) for feature in self.features_
-                )
+                ),
+                total=n_features,
             )
+        }
 
         self.results_ = MetaResult(self, mask=dataset.masker, maps=maps)
 

nimare/diagnostics.py

@@ -15,7 +15,7 @@
 from nimare.base import NiMAREBase
 from nimare.meta.cbma.base import PairwiseCBMAEstimator
 from nimare.meta.ibma import IBMAEstimator
-from nimare.utils import _check_ncores, get_masker, mm2vox, tqdm_joblib
+from nimare.utils import _check_ncores, get_masker, mm2vox
 
 LGR = logging.getLogger(__name__)
 
@@ -219,10 +219,16 @@ def transform(self, result):
             contribution_table = pd.DataFrame(index=rows, columns=cols)
             contribution_table.index.name = "id"
 
-            with tqdm_joblib(tqdm(total=len(meta_ids))):
-                contributions = Parallel(n_jobs=self.n_cores)(
-                    delayed(self._transform)(expid, label_map, sign, result) for expid in meta_ids
+            contributions = [
+                r
+                for r in tqdm(
+                    Parallel(return_as="generator", n_jobs=self.n_cores)(
+                        delayed(self._transform)(expid, label_map, sign, result)
+                        for expid in meta_ids
+                    ),
+                    total=len(meta_ids),
                 )
+            ]
 
             # Add results to table
             for expid, stat_prop_values in zip(meta_ids, contributions):

nimare/meta/cbma/ale.py

@@ -13,7 +13,7 @@
 from nimare.meta.kernel import ALEKernel
 from nimare.stats import null_to_p, nullhist_to_p
 from nimare.transforms import p_to_z
-from nimare.utils import _check_ncores, tqdm_joblib, use_memmap
+from nimare.utils import _check_ncores, use_memmap
 
 LGR = logging.getLogger(__name__)
 __version__ = _version.get_versions()["version"]
@@ -520,26 +520,34 @@ def _fit(self, dataset1, dataset2):
             shape=(self.n_iters, n_voxels),
         )
 
-        with tqdm_joblib(tqdm(total=self.n_iters)):
-            Parallel(n_jobs=self.n_cores)(
-                delayed(self._run_permutation)(i_iter, n_grp1, ma_arr, iter_diff_values)
-                for i_iter in range(self.n_iters)
+        _ = [
+            r
+            for r in tqdm(
+                Parallel(return_as="generator", n_jobs=self.n_cores)(
+                    delayed(self._run_permutation)(i_iter, n_grp1, ma_arr, iter_diff_values)
+                    for i_iter in range(self.n_iters)
+                ),
+                total=self.n_iters,
             )
+        ]
 
         # Determine p-values based on voxel-wise null distributions
         # I know that joblib probably preserves order of outputs, but I'm paranoid, so we track
         # the iteration as well and sort the resulting p-value array based on that.
-        with tqdm_joblib(tqdm(total=n_voxels)):
-            p_values, voxel_idx = zip(
-                *Parallel(n_jobs=self.n_cores)(
+        p_values, voxel_idx = tqdm(
+            zip(
+                *Parallel(return_as="generator", n_jobs=self.n_cores)(
                     delayed(self._alediff_to_p_voxel)(
                         i_voxel,
                         diff_ale_values[i_voxel],
                         iter_diff_values[:, i_voxel],
                     )
                     for i_voxel in range(n_voxels)
-                )
-            )
+                ),
+            ),
+            total=n_voxels,
+        )
+
         # Convert to an array and sort the p-values array based on the voxel index.
         p_values = np.array(p_values)[np.array(voxel_idx)]
 
@@ -801,13 +809,18 @@ def _fit(self, dataset):
             mode="w+",
             shape=(self.n_iters, stat_values.shape[0]),
         )
-        with tqdm_joblib(tqdm(total=self.n_iters)):
-            Parallel(n_jobs=self.n_cores)(
-                delayed(self._run_permutation)(
-                    i_iter, iter_xyzs[i_iter], iter_df, perm_scale_values
-                )
-                for i_iter in range(self.n_iters)
+        _ = [
+            r
+            for r in tqdm(
+                Parallel(return_as="generator", n_jobs=self.n_cores)(
+                    delayed(self._run_permutation)(
+                        i_iter, iter_xyzs[i_iter], iter_df, perm_scale_values
+                    )
+                    for i_iter in range(self.n_iters)
+                ),
+                total=self.n_iters,
             )
+        ]
 
         p_values, z_values = self._scale_to_p(stat_values, perm_scale_values)
 
@@ -876,15 +889,18 @@ def _scale_to_p(self, stat_values, scale_values):
 
         # I know that joblib probably preserves order of outputs, but I'm paranoid, so we track
         # the iteration as well and sort the resulting p-value array based on that.
-        with tqdm_joblib(tqdm(total=n_voxels)):
-            p_values, voxel_idx = zip(
-                *Parallel(n_jobs=self.n_cores)(
+        p_values, voxel_idx = tqdm(
+            zip(
+                *Parallel(return_as="generator", n_jobs=self.n_cores)(
                     delayed(self._scale_to_p_voxel)(
                         i_voxel, stat_values[i_voxel], scale_values[:, i_voxel]
                     )
                     for i_voxel in range(n_voxels)
                 )
-            )
+            ),
+            total=n_voxels,
+        )
+
         # Convert to an array and sort the p-values array based on the voxel index.
         p_values = np.array(p_values)[np.array(voxel_idx)]
 

nimare/meta/cbma/base.py

@@ -24,7 +24,6 @@
     _check_type,
     get_masker,
     mm2vox,
-    tqdm_joblib,
     vox2mm,
 )
 
@@ -506,13 +505,18 @@ def _compute_null_montecarlo(self, n_iters, n_cores):
         iter_xyzs = np.split(rand_xyz, rand_xyz.shape[1], axis=1)
         iter_df = self.inputs_["coordinates"].copy()
 
-        with tqdm_joblib(tqdm(total=n_iters)):
-            perm_histograms = Parallel(n_jobs=n_cores)(
-                delayed(self._compute_null_montecarlo_permutation)(
-                    iter_xyzs[i_iter], iter_df=iter_df
-                )
-                for i_iter in range(n_iters)
+        perm_histograms = [
+            r
+            for r in tqdm(
+                Parallel(return_as="generator", n_jobs=n_cores)(
+                    delayed(self._compute_null_montecarlo_permutation)(
+                        iter_xyzs[i_iter], iter_df=iter_df
+                    )
+                    for i_iter in range(n_iters)
+                ),
+                total=n_iters,
             )
+        ]
 
         perm_histograms = np.vstack(perm_histograms)
         self.null_distributions_["histweights_corr-none_method-montecarlo"] = np.sum(
@@ -720,17 +724,22 @@ def correct_fwe_montecarlo(
             # Define connectivity matrix for cluster labeling
             conn = ndimage.generate_binary_structure(rank=3, connectivity=1)
 
-            with tqdm_joblib(tqdm(total=n_iters)):
-                perm_results = Parallel(n_jobs=n_cores)(
-                    delayed(self._correct_fwe_montecarlo_permutation)(
-                        iter_xyzs[i_iter],
-                        iter_df=iter_df,
-                        conn=conn,
-                        voxel_thresh=ss_thresh,
-                        vfwe_only=vfwe_only,
-                    )
-                    for i_iter in range(n_iters)
+            perm_results = [
+                r
+                for r in tqdm(
+                    Parallel(return_as="generator", n_jobs=n_cores)(
+                        delayed(self._correct_fwe_montecarlo_permutation)(
+                            iter_xyzs[i_iter],
+                            iter_df=iter_df,
+                            conn=conn,
+                            voxel_thresh=ss_thresh,
+                            vfwe_only=vfwe_only,
+                        )
+                        for i_iter in range(n_iters)
+                    ),
+                    total=n_iters,
                 )
+            ]
 
             fwe_voxel_max, fwe_cluster_size_max, fwe_cluster_mass_max = zip(*perm_results)
 

nimare/meta/cbma/mkda.py

@@ -17,7 +17,7 @@
 from nimare.meta.utils import _calculate_cluster_measures
 from nimare.stats import null_to_p, one_way, two_way
 from nimare.transforms import p_to_z
-from nimare.utils import _check_ncores, tqdm_joblib, vox2mm
+from nimare.utils import _check_ncores, vox2mm
 
 LGR = logging.getLogger(__name__)
 __version__ = _version.get_versions()["version"]
@@ -849,18 +849,23 @@ def correct_fwe_montecarlo(self, result, voxel_thresh=0.001, n_iters=1000, n_cor
         # Define connectivity matrix for cluster labeling
         conn = ndimage.generate_binary_structure(rank=3, connectivity=1)
 
-        with tqdm_joblib(tqdm(total=n_iters)):
-            perm_results = Parallel(n_jobs=n_cores)(
-                delayed(self._run_fwe_permutation)(
-                    iter_xyz1=iter_xyzs1[i_iter],
-                    iter_xyz2=iter_xyzs2[i_iter],
-                    iter_df1=iter_df1,
-                    iter_df2=iter_df2,
-                    conn=conn,
-                    voxel_thresh=ss_thresh,
-                )
-                for i_iter in range(n_iters)
+        perm_results = [
+            r
+            for r in tqdm(
+                Parallel(return_as="generator", n_jobs=n_cores)(
+                    delayed(self._run_fwe_permutation)(
+                        iter_xyz1=iter_xyzs1[i_iter],
+                        iter_xyz2=iter_xyzs2[i_iter],
+                        iter_df1=iter_df1,
+                        iter_df2=iter_df2,
+                        conn=conn,
+                        voxel_thresh=ss_thresh,
+                    )
+                    for i_iter in range(n_iters)
+                ),
+                total=n_iters,
             )
+        ]
 
         del rand_idx1, rand_xyz1, iter_xyzs1
         del rand_idx2, rand_xyz2, iter_xyzs2

nimare/utils.py

@@ -1,6 +1,5 @@
 """Utility functions for NiMARE."""
 
-import contextlib
 import datetime
 import inspect
 import json
@@ -947,30 +946,6 @@ def _boolean_unmask(data_array, bool_array):
     return unmasked_data
 
 
-@contextlib.contextmanager
-def tqdm_joblib(tqdm_object):
-    """Context manager to patch joblib to report into tqdm progress bar given as argument.
-
-    From https://stackoverflow.com/a/58936697/2589328.
-    """
-
-    class TqdmBatchCompletionCallback(joblib.parallel.BatchCompletionCallBack):
-        def __init__(self, *args, **kwargs):
-            super().__init__(*args, **kwargs)
-
-        def __call__(self, *args, **kwargs):
-            tqdm_object.update(n=self.batch_size)
-            return super().__call__(*args, **kwargs)
-
-    old_batch_callback = joblib.parallel.BatchCompletionCallBack
-    joblib.parallel.BatchCompletionCallBack = TqdmBatchCompletionCallback
-    try:
-        yield tqdm_object
-    finally:
-        joblib.parallel.BatchCompletionCallBack = old_batch_callback
-        tqdm_object.close()
-
-
 def unique_rows(ar, return_counts=False):
     """Remove repeated rows from a 2D array.