config.yaml

# Configuration file for the image classification application
#
# Date: September 2018
# Author: Ignacio Heredia
# Email: iheredia@ifca.unican.es
# Github: ignacioheredia
#
# References
# ----------
# https://pyyaml.org/wiki/PyYAMLDocumentation


#####################################################
# Options for general configuration
#####################################################

general:

  base_directory:
    value: "."
    type: "str"
    help: >
          Base directory for data and models. All the data that will be read and written will be done within this
          directory.
          If path is relative it will be appended to the package path.

  images_directory:
    value: "/storage/Imagine_UC5_new/data/images"
    type: "str"
    help: >
          Base directory for images. If the path is relative, it will be appended to the package path.


#####################################################
#  Options to customize the model
#####################################################

model:

  modelname:
      value: "Xception"
      type: "str"
      choices: ["DenseNet121", "DenseNet169", "DenseNet201", "InceptionResNetV2", "InceptionV3", "MobileNet",
               "NASNetMobile", "Xception", "ResNet50", "VGG16", "VGG19"]
      help: >
            Model to train.
            A performance/size comparison can be found in: https://github.com/keras-team/keras-applications.
            "NASNetLarge" has been disabled as a choice as it gave compilation errors.

  image_size:
    value: 100
    type: "int"
    range: [32, 1000]
    help: >
          Image size to feed the model. If not provided, we will use the default image size of each model.

  num_classes:
    value:
    type: "int"
    range: [1, None]
    help: >
          Total number of possible output classes. If not provided, we will use the max label number from y_train.

  preprocess_mode:
    value:
    type: "str"
    help: >
          Mode of image preprocessing. It's value will depend on the model you select, therefore you don't need to set
          it.


#####################################################
#  Options about your dataset
#####################################################

dataset:
  mean_RGB:
    value: 
    type: "list"
    help: >
          Mean RGB value of the image dataset. If not provided it will be automatically computed before the training
          starts. Computing it can be quite time consuming, so it is useful to provide it if one if wishing to try
          different training configurations with the same dataset.
          Example: [158.33844060679408, 158.33844060679408, 158.33844060679408]

  std_RGB: 
    value: 
    type: "list"
    help: >
          Standard deviation of the RGB value of the image dataset. It is actually not needed for training.
          Example: [48.89589490647217, 48.89589490647217, 48.89589490647217]

  split_ratios: 
    value: [0.8, 0.1, 0.1]
    type: "list"
    help: >
          Split the dataset in [training, validation, test]

#####################################################
#  Options about your training routine
#####################################################

training:

  mode:
    value: "normal"
    type: "str"
    choices: ['normal', 'fast']
    help: >
      Mode for the training routine. In the `fast` mode we do not training the lower layers convolutional layers and
      only train the upper dense layer. The gain in training speed (~25% faster?) comes at the cost of a somewhat lower
      accuracy, because the weights of the convolutional layers will be the ones from the Imagenet pretraining and thus
      not finetuned to our dataset.

  initial_lr:
    value: 0.001
    type: "float"
    help: >
          Initial learning rate.

  batch_size:
    value: 16
    type: "int"
    range: [1, 64]
    help: >
          Batchsize to use during training. If your model has a large number of classes (>1000) you might need to decrease
          your batchsize so that the model still fits in the GPU.

  epochs:
    value: 15
    type: "int"
    range: [0, None]
    help: >
          Number of epochs to use for training.

  ckpt_freq:
    value:
    type: "float"
    range: [0, 1]
    help: >
          Frequency of the checkpoints (Float between 0 and 1). If None there will be no checkpoints saved. If 0.0 there
          will be 1 checkpoint per epoch. For example 0.1 means there will be 10 ckpts during the training.

  lr_schedule_mode:
    value: "step"
    type: "str"
    choices: ['step']
    help: >
          Mode for the learning rate schedule computation.

  lr_step_decay:
    value: 0.1
    type: "float"
    range: [0, 1]
    help: >
          Amount to decay the lr. Only relevant if lr_schedule_mode is set to 'step'

  lr_step_schedule:
    value: [0.7, 0.9]
    type: "list"
    item_type: float
    help: >
          List of the fraction of the total time at which apply a decay. For example [0.7, 0.9] means that the lr
          will be decay at 70% and 90% of total number of epochs.

  l2_reg:
    value: 0.0001
    type: "float"
    help: >
          L2 regularizer for the two last Dense layers.

  use_class_weights:
    value: False
    type: "bool"
    help: >
          Whether to use or not class_weights for the loss function computation. It's sometimes useful when the dataset
          classes are very imbalanced although it can make training more unstable.

  use_validation:
    value: True
    type: "bool"
    help: >
          Whether to use or not validation. If True you have to provide a `val.txt` file in the `splits` directory.

  use_test:
    value: True
    type: "bool"
    help: >
          Whether to use or not test. If True you have to provide a `test.txt` file in the `splits` directory.
          
  use_early_stopping:
    value: False
    type: "bool"
    help: >
          Whether to use or not early stopping. If True you have to provide a `val.txt` file in the `splits` directory.

  use_multiprocessing:
    value: False
    type: "bool"
    help: >
          Whether to use or not multiple workers to do preprocess data (ie. do data augmentation)
          faster during training. Disable it if your computing resources are scarce. Sometimes
          multiprocessing can give errors even with enough resources.

#####################################################
#  Options about monitoring your training
#####################################################

monitor:

  use_tensorboard:
    value: True
    type: "bool"
    help: >
          Use tensorboard to visualize the relevant metrics during the training process.

  use_remote:
    value: False
    type: "bool"
    help: >
          Forward the logs through a defined port if executing in a remote machine.


#####################################################
#  Options about your dataset image augmentation
#####################################################

augmentation:

  use_augmentation: 
    value: False
    type: "bool"
    help: >
          Use image augmentation during the training process.

  train_mode:
    value:
      h_flip: 0.5
      v_flip: 0.5
      rot: 0.7
      rot_lim: 90
      stretch: 0.0
      crop: 1.
      zoom: 0.2
      blur: 0.3
      pixel_noise: 0.3
      pixel_sat: 0.3
      cutout: 0.5
    help: >
          Augmentation parameters to use during the training phase. The meaning of the parameters is the following:
            - h_flip ([0,1] float): probability of performing an horizontal left-right mirroring.
            - v_flip ([0,1] float): probability of performing an vertical up-down mirroring.
            - rot ([0,1] float):  probability of performing a rotation to the image.
            - rot_lim (int):  max degrees of rotation.
            - stretch ([0,1] float):  probability of randomly stretching an image. It is set to zero by default because
              on some datasets it can lead to nonsensical images.
            - crop ([0,1] float): randomly take an image crop.
            - zoom ([0,1] float): random zoom applied to crop_size.
                --> Therefore the effective crop size at each iteration will be a
                    random number between 1 and crop*(1-zoom). For example:
                      * crop=1, zoom=0: no crop of the image
                      * crop=1, zoom=0.1: random crop of random size between 100% image and 90% of the image
                      * crop=0.9, zoom=0.1: random crop of random size between 90% image and 80% of the image
                      * crop=0.9, zoom=0: random crop of always 90% of the image
                      Image size refers to the size of the shortest side.
            - blur ([0,1] float):  probability of randomly blurring an image.
            - pixel_noise ([0,1] float):  probability of randomly adding pixel noise to an image.
            - pixel_sat ([0,1] float):  probability of randomly using HueSaturationValue in the image.
            - cutout ([0,1] float):  probability of using cutout in the image.

          Use an empty dict {} in case you want to disable data augmentation.

  val_mode:
    value:
      h_flip: 0.5
      v_flip: 0.
      rot: 0.5
      rot_lim: 30
      stretch: 0.
      crop: 0.9
      zoom: 0.1
      blur: 0.1
      pixel_noise: 0.1
      pixel_sat: 0.1
      cutout: 0.
    help: >
          Augmentation parameters to use during the validation/testing phase. The augmentation is smaller than in the
          case of training parameters. The meaning of the parameters is the same as in the train_mode option.

          Use an empty dict {} in case you want to disable data augmentation.

          


#####################################################
#  Options to test your model
#####################################################

testing:
  file_location:
    value: "/srv/phyto-plankton-classification/data/demo-images/Actinoptychus"
    type: "str"
    help: >
      Select the folder of the images you want to classify. For example: /storage/.../images_to_be_predicted

  timestamp:
    value: 
    type: "str"
    help: >
      Model timestamp to use for prediction.
      
  ckpt_name:
    value: "final_model.h5"
    type: "str"
    help: >
      Checkpoint inside the timestamp to use for prediction.

  output_directory:
    value: "/srv/somewhere"
    type: "str"
    help: >
      Location where to save the predictions. If empty, will save in mother folder of file_location folder.

  urls:
    value: 
    type: "str"
    help: >
      Select a URL of the image you want to classify.