296 Add memory usage to PyOPIA log output

pyopia/__init__.py

@@ -1 +1 @@
-__version__ = '2.8.1'
+__version__ = '2.8.2'

pyopia/background.py

@@ -56,8 +56,14 @@ def shift_bgstack_accurate(bgstack, imbg, imnew):
         updated actual background image
     '''
     bgstack.pop(0)  # pop the oldest image from the stack,
-    bgstack.append(imnew)  # append the new image to the stack
-    imbg = np.mean(bgstack, axis=0)
+
+    # Append the new image to the stack. Convert to uint8 from float [0-1],
+    # to save memory for large bgstacks, and speed up mean calculation of mean.
+    bgstack.append((255 * imnew).astype(np.uint8))
+
+    # Calculate mean image (float [0 - 1])
+    imbg = np.mean(bgstack, axis=0) / 255
+
     return bgstack, imbg
 
 
@@ -86,12 +92,20 @@ def shift_bgstack_fast(bgstack, imbg, imnew):
         updated actual background image
     '''
     stacklength = len(bgstack)
-    imold = bgstack.pop(0)  # pop the oldest image from the stack,
-    # subtract the old image from the average (scaled by the average window)
+
+    # Pop oldest image from stack, convert to float [0 - 1]
+    imold = bgstack.pop(0).astype(np.float64) / 255
+
+    # Subtract the old image from the average (scaled by the average window)
     imbg -= (imold / stacklength)
-    # add the new image to the average (scaled by the average window)
+
+    # Add the new image to the average (scaled by the average window)
     imbg += (imnew / stacklength)
-    bgstack.append(imnew)  # append the new image to the stack
+
+    # Append the new image to the stack. Convert to uint8 from float [0-1],
+    # to save memory for large bgstacks, and speed up mean calculation of mean.
+    bgstack.append((255 * imnew).astype(np.uint8))  # append the new image to the stack
+
     return bgstack, imbg
 
 
@@ -284,8 +298,13 @@ def _build_background_step(self, data):
 
         init_complete = True
         if len(data['bgstack']) < self.average_window:
-            data['bgstack'].append(data[self.image_source])
-            data['imbg'] = np.mean(data['bgstack'], axis=0)
+            # Append the new image to the stack. Convert to uint8 from float [0-1],
+            # to save memory for large bgstacks, and speed up mean calculation of mean.
+            img_uint8 = (255 * data[self.image_source]).astype(np.uint8)
+            data['bgstack'].append(img_uint8)
+
+            # Calculate background as mean image (float [0 - 1])
+            data['imbg'] = np.mean(data['bgstack'], axis=0) / 255
             init_complete = False
 
         return init_complete

pyopia/cli.py

@@ -14,7 +14,7 @@
 import rich.progress
 import pandas as pd
 import multiprocessing
-
+import psutil
 import pyopia
 import pyopia.background
 import pyopia.instrument.silcam
@@ -30,6 +30,15 @@
 app = typer.Typer()
 
 
+def log_memory_usage():
+    logger = logging.getLogger()
+    process = psutil.Process(os.getpid())
+    total = psutil.virtual_memory().total / 1e9
+    used = process.memory_info().rss / 1e9
+    used_percent = used / total * 100
+    logger.info(f'Current memory usage is: {used:.1f} / {total:.1f} GB ({used_percent:3.0f} %)')
+
+
 @app.command()
 def docs():
     '''Open browser at PyOPIA's readthedocs page
@@ -186,9 +195,12 @@ def process_file_list(file_list, c):
                                    f'(chunk {c})')
                     logger.error(e)
                     logger.debug(''.join(traceback.format_tb(e.__traceback__)))
+                finally:
+                    log_memory_usage()
 
     # With one chunk we keep the non-multiprocess functionality to ensure backwards compatibility
     job_list = []
+    multiprocessing.set_start_method('fork')
     if num_chunks == 1:
         process_file_list(raw_files, 0)
     else: