Evaluation.py

############################################################################################
#
# Project:       Peter Moss Acute Myeloid & Lymphoblastic Leukemia AI Research Project
# Repository:    ALL Detection System 2019
# Project:       Facial Authentication Server
#
# Author:        Adam Milton-Barker (AdamMiltonBarker.com)
# Contributors:
# Title:         Evaluation Class
# Description:   Evaluation class for the ALL Detection System 2019 NCS1 Classifier.
# License:       MIT License
# Last Modified: 2020-07-16
#
############################################################################################

import cv2, json, matplotlib, os, sys, time

import Classes.inception_preprocessing

import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import tkinter as tk
import pylab as pl

from tensorflow.python.framework import graph_util
from tensorflow.contrib.framework.python.ops.variables import get_or_create_global_step
from tensorflow.python.platform import tf_logging as logging

from Classes.Helpers import Helpers
from Classes.Data import Data
from Classes.inception_v3 import inception_v3, inception_v3_arg_scope

matplotlib.use("Agg")
plt.style.use('ggplot')
slim = tf.contrib.slim

# config = tf.ConfigProto(intra_op_parallelism_threads=12, inter_op_parallelism_threads=2,
#                        allow_soft_placement=True,  device_count={'CPU': 12})

#os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3'
#os.environ["OMP_NUM_THREADS"] = "12"
#os.environ["KMP_BLOCKTIME"] = "30"
#os.environ["KMP_SETTINGS"] = "1"
#os.environ["KMP_AFFINITY"] = "granularity=fine,verbose,compact,1,0"

class Evaluation():
    """ Evaluation Class

    Evaluates the ALL Detection System 2019 NCS1 Classifier.
    """

    def __init__(self):
        """ Initializes the Evaluation Class """

        self.Helpers = Helpers("Evaluator")
        self.confs = self.Helpers.confs

        self.labelsToName = {}

        self.checkpoint_file = tf.train.latest_checkpoint(
            self.confs["Classifier"]["LogDir"])

        # Open the labels file
        self.labels = open(
            self.confs["Classifier"]["DatasetDir"] + "/" + self.confs["Classifier"]["Labels"], 'r')

        # Create a dictionary to refer each label to their string name
        for line in self.labels:
            label, string_name = line.split(':')
            string_name = string_name[:-1]  # Remove newline
            self.labelsToName[int(label)] = string_name

        # Create a dictionary that will help people understand your dataset better. This is required by the Dataset class later.
        self.items_to_descriptions = {
            'image': 'A 3-channel RGB coloured  image that is ex: office, people',
            'label': 'A label that ,start from zero'
        }

        self.Helpers.logger.info(
            "Evaluator class initialization complete.")

    # ============== DATASET LOADING ======================
    # We now create a function that creates a Dataset class which will give us many TFRecord files to feed in the examples into a queue in parallel.
    def getSplit(self, split_name):
        '''
            Obtains the split - training or validation - to create a Dataset class for feeding the examples into a queue later on. This function will
            set up the decoder and dataset information all into one Dataset class so that you can avoid the brute work later on.
            Your file_pattern is very important in locating the files later.

            INPUTS:
                - split_name(str): 'train' or 'validation'. Used to get the correct data split of tfrecord files

            OUTPUTS:
                - dataset (Dataset): A Dataset class object where we can read its various components for easier batch creation later.
        '''

        # First check whether the split_name is train or validation
        if split_name not in ['train', 'validation']:

            raise ValueError(
                'The split_name %s is not recognized. Please input either train or validation as the split_name' % (split_name))

        # Create the full path for a general file_pattern to locate the tfrecord_files
        file_pattern_path = os.path.join(
            self.confs["Classifier"]["DatasetDir"], self.confs["Classifier"]["TFRecordPattern"] % (split_name))

        # Count the total number of examples in all of these shard
        num_samples = 0
        file_pattern_for_counting = 'ALL_' + split_name
        tfrecords_to_count = [os.path.join(self.confs["Classifier"]["DatasetDir"], file) for file in os.listdir(
            self.confs["Classifier"]["DatasetDir"]) if file.startswith(file_pattern_for_counting)]

        # print(tfrecords_to_count)
        for tfrecord_file in tfrecords_to_count:

            for record in tf.python_io.tf_record_iterator(tfrecord_file):

                num_samples += 1

        # Create a reader, which must be a TFRecord reader in this case
        reader = tf.TFRecordReader

        # Create the keys_to_features dictionary for the decoder
        keys_to_features = {
            'image/encoded': tf.FixedLenFeature((), tf.string, default_value=''),
            'image/format': tf.FixedLenFeature((), tf.string, default_value='jpg'),
            'image/class/label': tf.FixedLenFeature(
                [], tf.int64, default_value=tf.zeros([], dtype=tf.int64)),
        }

        # Create the items_to_handlers dictionary for the decoder.
        items_to_handlers = {
            'image': slim.tfexample_decoder.Image(),
            'label': slim.tfexample_decoder.Tensor('image/class/label'),
        }

        # Start to create the decoder
        decoder = slim.tfexample_decoder.TFExampleDecoder(
            keys_to_features, items_to_handlers)

        # Create the labels_to_name file
        labels_to_name_dict = self.labelsToName

        # Actually create the dataset
        dataset = slim.dataset.Dataset(
            data_sources=file_pattern_path,
            decoder=decoder,
            reader=reader,
            num_readers=4,
            num_samples=num_samples,
            num_classes=self.confs["Classifier"]["NumClasses"],
            labels_to_name=labels_to_name_dict,
            items_to_descriptions=self.items_to_descriptions)

        return dataset

    def loadBatch(self, dataset, is_training=True):
        '''
            Loads a batch for training.

            INPUTS:
                - dataset(Dataset): a Dataset class object that is created from the get_split function
                - batch_size(int): determines how big of a batch to train
                - height(int): the height of the image to resize to during preprocessing
                - width(int): the width of the image to resize to during preprocessing
                - is_training(bool): to determine whether to perform a training or evaluation preprocessing

            OUTPUTS:
                - images(Tensor): a Tensor of the shape (batch_size, height, width, channels) that contain one batch of images
                - labels(Tensor): the batch's labels with the shape (batch_size,) (requires one_hot_encoding).

        '''

        # First create the data_provider object
        data_provider = slim.dataset_data_provider.DatasetDataProvider(
            dataset,
            common_queue_capacity=24 + 3 *
            self.confs["Classifier"]["BatchTestSize"],
            common_queue_min=24)

        # Obtain the raw image using the get method
        raw_image, label = data_provider.get(['image', 'label'])

        # Perform the correct preprocessing for this image depending if it is training or evaluating
        image = Classes.inception_preprocessing.preprocess_image(
            raw_image, self.confs["Classifier"]["ImageSize"], self.confs["Classifier"]["ImageSize"], is_training)

        # As for the raw images, we just do a simple reshape to batch it up
        raw_image = tf.image.resize_image_with_crop_or_pad(
            raw_image, self.confs["Classifier"]["ImageSize"], self.confs["Classifier"]["ImageSize"])

        # Batch up the image by enqueing the tensors internally in a FIFO queue and dequeueing many elements with tf.train.batch.
        images, raw_images, labels = tf.train.batch(
            [image, raw_image, label],
            batch_size=self.confs["Classifier"]["BatchTestSize"],
            num_threads=4,
            capacity=4 * self.confs["Classifier"]["BatchTestSize"],
            allow_smaller_final_batch=True)

        return images, raw_images, labels


Evaluation = Evaluation()


def run():

    # Create LogDirEval for evaluation information
    if not os.path.exists(Evaluation.confs["Classifier"]["LogDirEval"]):
        os.mkdir(Evaluation.confs["Classifier"]["LogDirEval"])

    # Just construct the graph from scratch again
    with tf.Graph().as_default() as graph:

        tf.logging.set_verbosity(tf.logging.INFO)

        # Get the dataset first and load one batch of validation images and labels tensors. Set is_training as False so as to use the evaluation preprocessing
        dataset = Evaluation.getSplit('validation')
        images, raw_images, labels = Evaluation.loadBatch(dataset, is_training=False)

        # Create some information about the training steps
        num_batches_per_epoch = dataset.num_samples / \
            Evaluation.confs["Classifier"]["BatchTestSize"]
        num_steps_per_epoch = num_batches_per_epoch

        # Now create the inference model but set is_training=False
        with slim.arg_scope(inception_v3_arg_scope()):
            logits, end_points = inception_v3(
                images, num_classes=dataset.num_classes, is_training=False)

        # Perform one-hot-encoding of the labels (Try one-hot-encoding within the load_batch function!)
        one_hot_labels = slim.one_hot_encoding(labels, dataset.num_classes)

        # Performs the equivalent to tf.nn.sparse_softmax_cross_entropy_with_logits but enhanced with checks
        loss = tf.losses.softmax_cross_entropy(
            onehot_labels=one_hot_labels, logits=logits)
        # obtain the regularization losses as well
        total_loss = tf.losses.get_total_loss()

        # #get all the variables to restore from the checkpoint file and create the saver function to restore
        variables_to_restore = slim.get_variables_to_restore()
        saver = tf.train.Saver(variables_to_restore)

        def restore_fn(sess):
            return saver.restore(sess, Evaluation.checkpoint_file)

        # Just define the metrics to track without the loss or whatsoever
        probabilities = end_points['Predictions']
        predictions = tf.argmax(probabilities, 1)

        accuracy, accuracy_update = tf.contrib.metrics.streaming_accuracy(
            predictions, labels)
        metrics_op = tf.group(accuracy_update)

        # Create the global step and an increment op for monitoring
        global_step = get_or_create_global_step()
        # no apply_gradient method so manually increasing the global_step
        global_step_op = tf.assign(global_step, global_step + 1)

        # Create a evaluation step function
        def eval_step(sess, metrics_op, global_step):
            '''
            Simply takes in a session, runs the metrics op and some logging information.
            '''
            start_time = time.time()
            _, global_step_count, accuracy_value = sess.run(
                [metrics_op, global_step_op, accuracy])
            time_elapsed = time.time() - start_time

            # Log some information
            logging.info('Global Step %s: Streaming Accuracy: %.4f (%.2f sec/step)',
                         global_step_count, accuracy_value, time_elapsed)

            return accuracy_value

        # Define some scalar quantities to monitor
        tf.summary.scalar("Validation_Accuracy", accuracy)
        tf.summary.scalar("Validation_losses/Total_Loss", total_loss)
        my_summary_op = tf.summary.merge_all()

        # Get your supervisor
        sv = tf.train.Supervisor(
            logdir=Evaluation.confs["Classifier"]["LogDirEval"], summary_op=None, init_fn=restore_fn)

        # Now we are ready to run in one session
        with sv.managed_session() as sess:
            for step in range(int(num_batches_per_epoch * Evaluation.confs["Classifier"]["EpochsTest"])):
                # print vital information every start of the epoch as always
                if step % num_batches_per_epoch == 0:
                    logging.info('Epoch: %s/%s', step / num_batches_per_epoch + 1,
                                 Evaluation.confs["Classifier"]["EpochsTest"])
                    logging.info('Current Streaming Accuracy: %.4f',
                                 sess.run(accuracy))

                # Compute summaries every 10 steps and continue evaluating
                if step % 10 == 0:
                    eval_step(sess, metrics_op=metrics_op,
                              global_step=sv.global_step)
                    summaries = sess.run(my_summary_op)
                    sv.summary_computed(sess, summaries)

                # Otherwise just run as per normal
                else:
                    eval_step(sess, metrics_op=metrics_op,
                              global_step=sv.global_step)

            # At the end of all the evaluation, show the final accuracy
            logging.info('Final Streaming Accuracy: %.4f', sess.run(accuracy))

            # Now we want to visualize the last batch's images just to see what our model has predicted
            raw_images, labels, predictions, probabilities = sess.run(
                [raw_images, labels, predictions, probabilities])
            for i in range(10):
                image, label, prediction, probability = raw_images[
                    i], labels[i], predictions[i], probabilities[i]
                prediction_name, label_name = dataset.labels_to_name[
                    prediction], dataset.labels_to_name[label]
                text = 'Prediction: %s \n Ground Truth: %s \n Probability: %s' % (
                    prediction_name, label_name, probability[prediction])
                img_plot = plt.imshow(image)

                # Set up the plot and hide axes
                # plt.title(text)
                # img_plot.axes.get_yaxis().set_ticks([])
                # img_plot.axes.get_xaxis().set_ticks([])
                # plt.show()

            logging.info(
                'Model evaluation has completed! Visit TensorBoard for more information regarding your evaluation.')
            sv.saver.save(sess, sv.save_path, global_step=sv.global_step)


if __name__ == '__main__':
    run()