Merge pull request #80 from Gregor-Mendel-Institute/development

Retraining option via container added and some debug

.gitignore

@@ -4,21 +4,23 @@ datasets
 RUNID
 model_eval
 laptop_debug.sh
+cluster_debug.sh
 laptop_docker_test.sh
 laptop_singularity_test.sh
 trg-imageprocessing_master.sif
 image-processing_master.sif
-cluster_time_images.sh
+
 
 Dockerfile_with_Cracks
 Docker_commands
 TRG-process.def
-
+*.pkl
 *.pyc
 *.ipynb
 **.pyc
 **.stdout
 __pycache__/
 postprocessingCracksRings_backup.py
 postprocessingCracksRings_test.py
+postprocessingFunctions.py
 Train_onlyRing_test.sh

CoreProcessingPipelineScripts/CNN/Mask_RCNN/DetectionConfig/TreeRing_onlyCracks.py

@@ -26,7 +26,7 @@
 
 # Root directory of the project
 ROOT_DIR = os.path.abspath("../../")
-print(ROOT_DIR)
+#print(ROOT_DIR)
 # Import Mask RCNN
 sys.path.append(ROOT_DIR)  # To find local version of the library
 from mrcnn.config import Config

CoreProcessingPipelineScripts/CNN/Mask_RCNN/DetectionConfig/TreeRing_onlyRing.py

@@ -27,7 +27,7 @@
 
 # Root directory of the project
 ROOT_DIR = os.path.abspath("../../")
-print(ROOT_DIR)
+#print(ROOT_DIR)
 # Import Mask RCNN
 sys.path.append(ROOT_DIR)  # To find local version of the library
 from mrcnn.config import Config

CoreProcessingPipelineScripts/CNN/Mask_RCNN/Singularity.def

@@ -0,0 +1,30 @@
+Bootstrap: docker
+From: mambaorg/micromamba:0.20.0
+Stage: spython-base
+
+%files
+. /home/mambauser/
+%post
+
+
+mkdir -p /home/mambauser
+cd /home/mambauser
+
+micromamba install -y -n base wget -f /home/mambauser/environment.yml && micromamba clean -y -a
+
+MAMBA_DOCKERFILE_ACTIVATE=1
+
+wget https://data.swarts.gmi.oeaw.ac.at/treeringcrackscomb2_onlyring20210121T1457/mask_rcnn_treeringcrackscomb2_onlyring_0186.h5
+
+PATH="/home/mambauser/postprocessing:$MAMBA_ROOT_PREFIX/bin:$PATH"
+PYTHONPATH="/home/mambauser:$PYTHONPATH"
+
+%environment
+export PATH="/home/mambauser/postprocessing:$MAMBA_ROOT_PREFIX/bin:$PATH"
+export PYTHONPATH="/home/mambauser:$PYTHONPATH"
+%runscript
+cd /home/mambauser
+exec postprocessingCracksRings.py --weightRing /home/mambauser/mask_rcnn_treeringcrackscomb2_onlyring_0186.h5 "$@"
+%startscript
+cd /home/mambauser
+exec postprocessingCracksRings.py --weightRing /home/mambauser/mask_rcnn_treeringcrackscomb2_onlyring_0186.h5 "$@"

CoreProcessingPipelineScripts/CNN/Mask_RCNN/mrcnn/config.py

@@ -9,7 +9,6 @@
 
 import numpy as np
 
-
 # Base Configuration Class
 # Don't use this class directly. Instead, sub-class it and override
 # the configurations you need to change.

CoreProcessingPipelineScripts/CNN/Mask_RCNN/mrcnn/model.py

@@ -2092,7 +2092,7 @@ def find_last(self):
         checkpoint = os.path.join(dir_name, checkpoints[-1])
         return checkpoint
 
-    def load_weights(self, filepath, by_name=False, exclude=None):
+    def load_weights(self, filepath, by_name=False, exclude=None, start_new=False):
         """Modified version of the corresponding Keras function with
         the addition of multi-GPU support and the ability to exclude
         some layers from loading.
@@ -2134,7 +2134,7 @@ def load_weights(self, filepath, by_name=False, exclude=None):
             f.close()
 
         # Update the log directory
-        self.set_log_dir(filepath)
+        self.set_log_dir(filepath, start_new=start_new)
 
     def get_imagenet_weights(self):
         """Downloads ImageNet trained weights from Keras.
@@ -2235,7 +2235,7 @@ def set_trainable(self, layer_regex, keras_model=None, indent=0, verbose=1):
                 log("{}{:20}   ({})".format(" " * indent, layer.name,
                                             layer.__class__.__name__))
 
-    def set_log_dir(self, model_path=None):
+    def set_log_dir(self, model_path=None, start_new=False):
         """Sets the model log directory and epoch counter.
 
         model_path: If None, or a format different from what this code uses
@@ -2248,7 +2248,7 @@ def set_log_dir(self, model_path=None):
         now = datetime.datetime.now()
 
         # If we have a model path with date and epochs use them
-        if model_path:
+        if model_path and start_new==False:
             # Continue from we left of. Get epoch and date from the file name
             # A sample model path might look like:
             # \path\to\logs\coco20171029T2315\mask_rcnn_coco_0001.h5 (Windows)
@@ -2336,15 +2336,25 @@ def train(self, train_dataset, val_dataset, learning_rate, epochs, layers,
             os.makedirs(self.log_dir)
 
         # Callbacks
-        callbacks = [
-            keras.callbacks.TensorBoard(log_dir=self.log_dir,
-                                        histogram_freq=0, write_graph=True, write_images=False),
-            keras.callbacks.ModelCheckpoint(self.checkpoint_path,
-                                            verbose=0, save_weights_only=True),
-        ]
+        # modified for retraining so its saving only the best weights and does not take over hard drive
+        # it monitors val_loss by default so no need to specify
+        if custom_callbacks=="only_best":
+            callbacks = [
+                keras.callbacks.TensorBoard(log_dir=self.log_dir,
+                                            histogram_freq=0, write_graph=True, write_images=False),
+                keras.callbacks.ModelCheckpoint(self.checkpoint_path,
+                                                verbose=0, save_best_only=True, save_weights_only=True),
+            ]
+        else:
+            callbacks = [
+                keras.callbacks.TensorBoard(log_dir=self.log_dir,
+                                            histogram_freq=0, write_graph=True, write_images=False),
+                keras.callbacks.ModelCheckpoint(self.checkpoint_path,
+                                                verbose=0, save_weights_only=True),
+            ]
 
         # Add custom callbacks to the list
-        if custom_callbacks:
+        if isinstance(custom_callbacks, list):
             callbacks += custom_callbacks
 
         # Train

CoreProcessingPipelineScripts/CNN/Mask_RCNN/mrcnn/utils.py

@@ -25,7 +25,6 @@
 # URL from which to download the latest COCO trained weights
 COCO_MODEL_URL = "https://github.com/matterport/Mask_RCNN/releases/download/v2.0/mask_rcnn_coco.h5"
 
-
 ############################################################
 #  Bounding Boxes
 ############################################################

CoreProcessingPipelineScripts/CNN/Mask_RCNN/mrcnn/visualize_print.py

@@ -18,7 +18,7 @@
 import matplotlib.pyplot as plt
 from matplotlib import patches,  lines
 from matplotlib.patches import Polygon
-import IPython.display
+#import IPython.display
 
 # Root directory of the project
 ROOT_DIR = os.path.abspath("../")
@@ -462,11 +462,11 @@ def draw_boxes(image, boxes=None, refined_boxes=None,
                 ax.add_patch(p)
     ax.imshow(masked_image.astype(np.uint8))
 
-
+"""
+# i think this will not be missing
 def display_table(table):
-    """Display values in a table format.
-    table: an iterable of rows, and each row is an iterable of values.
-    """
+    #Display values in a table format.
+    #table: an iterable of rows, and each row is an iterable of values.
     html = ""
     for row in table:
         row_html = ""
@@ -475,8 +475,7 @@ def display_table(table):
         html += "<tr>" + row_html + "</tr>"
     html = "<table>" + html + "</table>"
     IPython.display.display(IPython.display.HTML(html))
-
-
+"""
 def display_weight_stats(model):
     """Scans all the weights in the model and returns a list of tuples
     that contain stats about each weight.

CoreProcessingPipelineScripts/CNN/Mask_RCNN/postprocessing/inference_cluster_debug.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+#SBATCH --nodes=1
+#SBATCH --partition=g
+#SBATCH --gres=gpu:RTX:1
+#SBATCH --mem=32G
+#SBATCH --qos=short
+#SBATCH --time=00-04:00:00
+#SBATCH --output=Inference_debug.stdout
+
+ml load anaconda3/2019.03
+source activate ~/.conda/envs/TreeRingCNN
+
+~/.conda/envs/TreeRingCNN/bin/python3 postprocessingCracksRings.py \
+  --dpi=13039 \
+  --run_ID=Inference_debug \
+  --input=/groups/swarts/user/miroslav.polacek/TRG-ImageProcessingTESTING/CoreProcessingPipelineScripts/CNN/Mask_RCNN/Data/General_testing_images \
+  --weightRing=/users/miroslav.polacek/TRG_testing/TRG-ImageProcessing/CoreProcessingPipelineScripts/CNN/Mask_RCNN/logs/onlyring/mask_rcnn_treeringcrackscomb2_onlyring_0186.h5 \
+  --output_folder=../output \
+  --print_detections=yes \