abandon MPIPoolExecutor

Author: moustakas

py/fastspecfit/mpi.py

@@ -283,14 +283,12 @@ def plan(size=1, specprod=None, specprod_dir=None, coadd_type='healpix',
 
     if merge:
         if len(outfiles) == 0:
-            if rank == 0:
-                log.debug(f'No {outprefix} files in {outdir} found!')
+            log.debug(f'No {outprefix} files in {outdir} found!')
             return '', list(), list(), list(), None
         return outdir, redrockfiles, outfiles, None, None
     elif makeqa:
         if len(outfiles) == 0:
-            if rank == 0:
-                log.debug(f'No {outprefix} files in {outdir} left to do!')
+            log.debug(f'No {outprefix} files in {outdir} left to do!')
             return '', list(), list(), list(), None
         #  hack--build the output directories and pass them in the 'redrockfiles'
         #  position! for coadd_type==cumulative, strip out the 'lastnight' argument
@@ -300,8 +298,7 @@ def plan(size=1, specprod=None, specprod_dir=None, coadd_type='healpix',
             redrockfiles = np.array([os.path.dirname(outfile).replace(outdir, htmldir) for outfile in outfiles])
     else:
         if len(redrockfiles) == 0:
-            if rank == 0:
-                log.info('All files have been processed!')
+            log.info('All files have been processed!')
             return '', list(), list(), list(), None
 
     return outdir, redrockfiles, outfiles, groups, ntargets

py/fastspecfit/util.py

@@ -53,22 +53,23 @@ class MPPool(object):
     rather than a list of positional arguments.
 
     """
-    def __init__(self, nworkers, initializer=None, init_argdict=None, comm=None):
+    def __init__(self, nworkers, initializer=None, init_argdict=None):
         """
         create a pool with nworkers workers, using the current
         process if nworkers is 1.  If initiializer is not None,
         apply this function to the arguments in keyword dictionary
         init_argdict on startup in each each worker subprocess.
 
         """
-
         initfunc = None if initializer is None else self.apply_to_dict
 
+        ## If multiprocessing, create a pool of worker processes and initialize
+        ## single-copy objects in each worker.
+        #if args.mp > 1 and not 'NERSC_HOST' in os.environ:
+        #    import multiprocessing
+        #    multiprocessing.set_start_method('fork')
         if nworkers > 1:
-            if comm is not None:
-                from mpi4py.futures import MPIPoolExecutor as Pool
-            else:
-                from multiprocessing import Pool
+            from multiprocessing import Pool
             self.pool = Pool(nworkers,
                              initializer=initfunc,
                              initargs=(initializer, init_argdict,))