Allow user-defined fields for image correlation

examples/blocks/dis_correl/dis_correl_custom_field.py

@@ -0,0 +1,107 @@
+# coding: utf-8
+
+"""
+This example demonstrates the use of the DISCorrel Block with user-provided
+fields. It does not require any hardware to run, but necessitates the
+opencv-python, matplotlib and Pillow modules to be installed.
+
+This Block computes several fields on acquired images by performing dense
+inverse search on a given patch. It outputs the averages of the computed
+fields over the entire patch. The fields can be automatically generated, or
+provided by the user.
+
+In this example, a fake strain is generated on a static image of a sample with
+a speckle. The level of strain is controlled by a Generator Block, and applied
+to the images by the DISCorrel Block. This same DISCorrel Block then calculates
+the strain on the images, based on both automatically-generated and
+user-provided fields, and outputs it to a Grapher Block for display.
+
+After starting this script, the patch appears in the configuration window. Do
+not re-select a patch in the configuration window ! Instead, close the
+configuration window to start the test, and watch the strain be calculated in
+real time. This demo normally ends automatically after 2 minutes. You can also
+hit CTRL+C to stop it earlier, but it is not a clean way to stop Crappy.
+"""
+
+import crappy
+import numpy as np
+
+if __name__ == '__main__':
+
+  # Loading the example image of a speckle used for performing the image
+  # correlation. This image is distributed with Crappy
+  img = crappy.resources.speckle
+
+  # Building the custom fields to use for the correlation, as numpy arrays of
+  # shape (patch_height, patch_width, 2)
+  # Here, for the sake of demonstration, they correspond to the 'exx' and 'eyy'
+  # fields that can be automatically generated by Crappy
+  width = 128
+  height = 128
+  exx = np.stack((np.tile(np.linspace(-width / 200, width / 200, width,
+                                      dtype=np.float32), (height, 1)),
+                  np.zeros((height, width), dtype=np.float32)), axis=2)
+  eyy = np.stack((np.zeros((height, width), dtype=np.float32),
+                  np.swapaxes(np.tile(np.linspace(-height / 200, height / 200,
+                                                  height, dtype=np.float32),
+                                      (width, 1)), 1, 0)), axis=2)
+
+  # The Generator Block that drives the fake strain on the image
+  # It applies a cyclic strain that makes the image stretch in the x direction
+  gen = crappy.blocks.Generator(
+      # Using a CyclicRamp Path to generate cyclic linear stretching
+      ({'type': 'CyclicRamp',
+        'speed1': 1,  # Stretching at 1%/s
+        'speed2': -1,  # Relaxing at 1%/s
+        'condition1': 'Exx(%)>20',  # Stretching until 20% strain
+        'condition2': 'Exx(%)<0',  # Relaxing until 0% strain
+        'cycles': 3,  # The test stops after 3 cycles
+        'init_value': 0},),  # Mandatory to give as it's the first Path
+      freq=50,  # Lowering the default frequency because it's just a demo
+      cmd_label='Exx(%)',  # The generated signal corresponds to a strain
+
+      # Sticking to default for the other arguments
+  )
+
+  # This DISCorrel Block calculates the strain of the image by performing dense
+  # inverse search on the given patch
+  # This Block is actually also the one that generates the fake strain on the
+  # image, but that wouldn't be the case in real life
+  # It takes the target strain as an input, and outputs the computed strain
+  # Some fields to use are user-provided, but correspond to what would be
+  # automatically generated by providing the value ('x', 'exx', 'eyy')
+  disco = crappy.blocks.DISCorrel(
+      '',  # The name of Camera to open is ignored because image_generator is
+      # given
+      config=True,  # Displaying the configuration window before starting,
+      # mandatory if the patches to track ar not given as arguments
+      display_images=True,  # The displayer window will allow to follow the
+      # patches on the speckle image
+      freq=50,  # Lowering the default frequency because it's just a demo
+      save_images=False,  # We don't want images to be recorded in this demo
+      image_generator=crappy.tool.ApplyStrainToImage(img),  # This argument
+      # makes the Block generate fake strain on the given image, only useful
+      # for demos
+      patch=(193, 193, height, width),  # Providing here the patch to follow
+      fields=('x', 'y', exx, eyy),  # The fields to compute on the acquired
+      # images, some provided by the user as numpy arrays, some generated
+      # automatically from strings
+      # The labels for sending the calculated strain to downstream Blocks
+      labels=('t(s)', 'meta', 'disp_x', 'disp_y', 'Exx(%)', 'Eyy(%)'),
+
+      # Sticking to default for the other arguments
+  )
+
+  # This Grapher displays the extension as computed by the DISCorrel Block
+  graph = crappy.blocks.Grapher(('t(s)', 'Exx(%)'))
+
+  # Linking the Blocks together so that each one sends and received the correct
+  # information
+  # The Generator drives the DISCorrel, but also takes decision based on its
+  # feedback
+  crappy.link(gen, disco)
+  crappy.link(disco, gen)
+  crappy.link(disco, graph)
+
+  # Mandatory line for starting the test, this call is blocking
+  crappy.start()

src/crappy/blocks/camera_processes/dis_correl.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 
 import numpy as np
-from typing import Optional, List
+from typing import Optional, List, Union
 import logging
 import logging.handlers
 
@@ -28,7 +28,7 @@ class DISCorrelProcess(CameraProcess):
 
   def __init__(self,
                patch: Box,
-               fields: Optional[List[str]] = None,
+               fields: Optional[List[Union[str, np.ndarray]]] = None,
                alpha: float = 3,
                delta: float = 1,
                gamma: float = 0,
@@ -47,15 +47,25 @@ def __init__(self,
         the coordinates of the ROI to perform DIS on. This argument is passed
         to the :obj:`~crappy.tool.image_processing.DISCorrelTool` and not used
         in this class.
-      fields: The base of fields to use for the projection, given as a
-        :obj:`list` of :obj:`str`. The available fields are :
+      fields: fields: The base of fields to use for the projection, given as a
+        :obj:`list` of :obj:`str` or :mod:`numpy` arrays (both types can be
+        mixed). Strings are for using automatically-generated fields, the
+        available ones are :
         ::
 
           'x', 'y', 'r', 'exx', 'eyy', 'exy', 'eyx', 'exy2', 'z'
 
+        If users provide their own fields as arrays, they will be used as-is to
+        run the correlation. The user-provided fields must be of shape:
+        ::
+
+          (patch_height, patch_width, 2)
+
         This argument is passed to the
         :obj:`~crappy.tool.image_processing.DISCorrelTool` and not used in this
         class.
+
+        .. versionchanged:: 2.0.5 provided fields can now be numpy arrays
       alpha: Weight of the smoothness term in DISFlow, as a :obj:`float`. This
         argument is passed to the
         :obj:`~crappy.tool.image_processing.DISCorrelTool` and not used in this

src/crappy/blocks/camera_processes/gpu_correl.py

@@ -37,7 +37,7 @@ def __init__(self,
                resampling_factor: float = 2,
                kernel_file: Optional[Union[str, Path]] = None,
                iterations: int = 4,
-               fields: Optional[List[str]] = None,
+               fields: Optional[List[Union[str, np.ndarray]]] = None,
                mask: Optional[np.ndarray] = None,
                mul: float = 3) -> None:
     """Sets the arguments and initializes the parent class.
@@ -88,16 +88,25 @@ def __init__(self,
         increasing. This argument is passed to the
         :class:`~crappy.tool.image_processing.GPUCorrelTool` and not used in
         this class.
-      fields: A :obj:`list` of :obj:`str` representing the base of fields on
-        which the image will be projected during correlation. The possible
-        fields are :
+      fields: The base of fields to use for the projection, given as a
+        :obj:`list` of :obj:`str` or :mod:`numpy` arrays (both types can be
+        mixed). Strings are for using automatically-generated fields, the
+        available ones are :
         ::
 
           'x', 'y', 'r', 'exx', 'eyy', 'exy', 'eyx', 'exy2', 'z'
 
+        If users provide their own fields as arrays, they will be used as-is to
+        run the correlation. The user-provided fields must be of shape:
+        ::
+
+          (patch_height, patch_width, 2)
+
         This argument is passed to the
         :class:`~crappy.tool.image_processing.GPUCorrelTool` and not used in
         this class.
+
+        .. versionchanged:: 2.0.5 provided fields can now be numpy arrays
       mask: The mask used for weighting the region of interest on the image. It
         is generally used to prevent unexpected behavior on the border of the
         image. This argument is passed to the

src/crappy/tool/image_processing/dis_correl.py

@@ -1,6 +1,7 @@
 # coding: utf-8
 
-from typing import List, Optional
+from typing import List, Optional, Union
+import numpy as np
 
 from ..._global import OptionalModule
 from ..camera_config import Box
@@ -10,7 +11,6 @@
   import cv2
 except (ModuleNotFoundError, ImportError):
   cv2 = OptionalModule("opencv-python")
-import numpy as np
 
 
 class DISCorrelTool:
@@ -27,7 +27,7 @@ class DISCorrelTool:
 
   def __init__(self,
                box: Box,
-               fields: Optional[List[str]] = None,
+               fields: Optional[List[Union[str, np.ndarray]]] = None,
                alpha: float = 3,
                delta: float = 1,
                gamma: float = 0,
@@ -46,11 +46,20 @@ def __init__(self,
 
         .. versionadded:: 2.0.0
       fields: The base of fields to use for the projection, given as a
-        :obj:`list` of :obj:`str`. The available fields are :
+        :obj:`list` of :obj:`str` or :mod:`numpy` arrays (both types can be
+        mixed). Strings are for using automatically-generated fields, the
+        available ones are :
         ::
 
           'x', 'y', 'r', 'exx', 'eyy', 'exy', 'eyx', 'exy2', 'z'
 
+        If users provide their own fields as arrays, they will be used as-is to
+        run the correlation. The user-provided fields must be of shape:
+        ::
+
+          (patch_height, patch_width, 2)
+
+        .. versionchanged:: 2.0.5 provided fields can now be numpy arrays
       alpha: Weight of the smoothness term in DISFlow, as a :obj:`float`.
       delta: Weight of the color constancy term in DISFlow, as a :obj:`float`.
       gamma: Weight of the gradient constancy term in DISFlow , as a
@@ -72,11 +81,26 @@ def __init__(self,
         less than patch size.
     """
 
-    if fields is not None and not all((field in allowed_fields
-                                       for field in fields)):
-      raise ValueError(f"The only allowed values for the fields "
-                       f"are {allowed_fields}")
-    self._fields = ["x", "y", "exx", "eyy"] if fields is None else fields
+    if fields is not None:
+      # Splitting the given fields into strings and numpy arrays
+      auto_fields = [field for field in fields if isinstance(field, str)]
+      user_fields = [field for field in fields
+                     if isinstance(field, np.ndarray)]
+
+      # Ensuring all the given fields are either strings or numpy arrays
+      if len(fields) != len(auto_fields) + len(user_fields):
+        raise TypeError('Correlation fields must be either strings or '
+                        'numpy arrays !')
+
+      # Ensuring all the string fields are valid ones
+      if not all((field in allowed_fields for field in auto_fields)):
+        raise ValueError(f"The only allowed values for the fields given as "
+                         f"strings are {allowed_fields}")
+
+      self._fields: List[Union[str, np.ndarray]] = fields
+    else:
+      self._fields: List[Union[str, np.ndarray]] = ["x", "y", "exx", "eyy"]
+
     self._init = init
 
     # These attributes will be set later
@@ -124,9 +148,12 @@ def set_box(self) -> None:
     # Creates and populates the base fields to use for correlation
     fields = np.empty((box_height, box_width, 2, len(self._fields)),
                       dtype=np.float32)
-    for i, string in enumerate(self._fields):
-      fields[:, :, 0, i], fields[:, :, 1, i] = get_field(string, box_height,
-                                                         box_width)
+    for i, field in enumerate(self._fields):
+      if isinstance(field, str):
+        fields[:, :, 0, i], fields[:, :, 1, i] = get_field(field, box_height,
+                                                           box_width)
+      elif isinstance(field, np.ndarray):
+        fields[:, :, :, i] = field
 
     # These attributes will be used later
     self._base = [fields[:, :, :, i] for i in range(fields.shape[3])]

src/crappy/tool/image_processing/gpu_correl.py

@@ -524,7 +524,7 @@ def __init__(self,
                resampling_factor: float = 2,
                kernel_file: Optional[Union[str, Path]] = None,
                iterations: int = 4,
-               fields: Optional[List[str]] = None,
+               fields: Optional[List[Union[str, np.ndarray]]] = None,
                ref_img: Optional[np.ndarray] = None,
                mask: Optional[np.ndarray] = None,
                mul: float = 3) -> None:
@@ -556,13 +556,21 @@ def __init__(self,
       iterations: The maximum number of iterations to run before returning the
         results. The results may be returned before if the residuals start
         increasing.
-      fields: A :obj:`list` of :obj:`str` representing the base of fields on
-        which the image will be projected during correlation. The possible
-        fields are :
+      fields: The base of fields to use for the projection, given as a
+        :obj:`list` of :obj:`str` or :mod:`numpy` arrays (both types can be
+        mixed). Strings are for using automatically-generated fields, the
+        available ones are :
         ::
 
           'x', 'y', 'r', 'exx', 'eyy', 'exy', 'eyx', 'exy2', 'z'
 
+        If users provide their own fields as arrays, they will be used as-is to
+        run the correlation. The user-provided fields must be of shape:
+        ::
+
+          (patch_height, patch_width, 2)
+
+        .. versionchanged:: 2.0.5 provided fields can now be numpy arrays
       ref_img: The reference image, as a 2D :obj:`numpy.array` with `dtype`
         `float32`. It can either be given at :meth:`__init__`, or set later
         with :meth:`set_orig`.
@@ -819,11 +827,16 @@ def _set_fields(self, fields: List[str]) -> None:
     """Computes the fields based on the provided field strings, and sets them
     for each stage."""
 
-    for field_str, tex_fx, tex_fy in zip(fields, self._tex_fx, self._tex_fy):
+    for field, tex_fx, tex_fy in zip(fields, self._tex_fx, self._tex_fy):
 
       # Getting the fields as numpy arrays
-      field_x, field_y = get_field(field_str, self._heights[0],
-                                   self._widths[0])
+      if isinstance(field, str):
+        field_x, field_y = get_field(field, self._heights[0], self._widths[0])
+      elif isinstance(field, np.ndarray):
+        field_x, field_y = field[:, :, 0], field[:, :, 1]
+      else:
+        raise TypeError("The provided fields should either be strings or "
+                        "numpy arrays !")
 
       tex_fx.set_array(pycuda.driver.matrix_to_array(field_x, 'C'))
       tex_fy.set_array(pycuda.driver.matrix_to_array(field_y, 'C'))