Merge pull request #7 from twuilliam/guidedbp

Adding guided backprop

Author: jacobgil

README.md

@@ -9,9 +9,27 @@ The paper authors torch implementation: https://github.com/ramprs/grad-cam
 
 This code assumes Tensorflow dimension ordering, and uses the VGG16 network in keras.applications by default (the network weights will be downloaded on first use).
 
-TODO: Combine with guided back propagation like in the paper.
-
 
 Usage: `python grad-cam.py <path_to_image>`
 
-![enter image description here](https://github.com/jacobgil/keras-grad-cam/blob/master/examples/boat.jpg?raw=true) ![enter image description here](https://github.com/jacobgil/keras-grad-cam/blob/master/examples/persian_cat.jpg?raw=true)
+
+##### Examples
+
+![enter image description here](https://github.com/jacobgil/keras-grad-cam/blob/master/examples/boat.jpg?raw=true) ![enter image description here](https://github.com/jacobgil/keras-grad-cam/blob/master/examples/persian_cat.jpg?raw=true)
+
+Example image from the [original implementation](https://github.com/ramprs/grad-cam):  
+
+'boxer' (243 or 242 in keras)
+
+![](/examples/cat_dog.png)
+![](/examples/cat_dog_242_gradcam.jpg)
+![](/examples/cat_dog_242_guided_gradcam.jpg)
+
+'tiger cat' (283 or 282 in keras)
+
+![](/examples/cat_dog.png)
+![](/examples/cat_dog_282_gradcam.jpg)
+![](/examples/cat_dog_282_guided_gradcam.jpg)
+
+
+

examples/cat_dog.png

@@ -1,16 +1,18 @@
-from keras.applications.vgg16 import VGG16
+from keras.applications.vgg16 import (
+    VGG16, preprocess_input, decode_predictions)
 from keras.preprocessing import image
-from keras.applications.vgg16 import preprocess_input
 from keras.layers.core import Lambda
 from keras.models import Sequential
+from tensorflow.python.framework import ops
 import keras.backend as K
 import tensorflow as tf
 import numpy as np
+import keras
 import sys
 import cv2
 
 def target_category_loss(x, category_index, nb_classes):
-    return tf.mul(x, K.one_hot([category_index], nb_classes))
+    return tf.multiply(x, K.one_hot([category_index], nb_classes))
 
 def target_category_loss_output_shape(input_shape):
     return input_shape
@@ -27,7 +29,63 @@ def load_image(path):
     x = preprocess_input(x)
     return x
 
-def grad_cam(input_model, image, category_index, layer_name): 
+def register_gradient():
+    if "GuidedBackProp" not in ops._gradient_registry._registry:
+        @ops.RegisterGradient("GuidedBackProp")
+        def _GuidedBackProp(op, grad):
+            dtype = op.inputs[0].dtype
+            return grad * tf.cast(grad > 0., dtype) * \
+                tf.cast(op.inputs[0] > 0., dtype)
+
+def compile_saliency_function(model, activation_layer='block5_conv3'):
+    input_img = model.input
+    layer_dict = dict([(layer.name, layer) for layer in model.layers[1:]])
+    layer_output = layer_dict[activation_layer].output
+    max_output = K.max(layer_output, axis=3)
+    saliency = K.gradients(K.sum(max_output), input_img)[0]
+    return K.function([input_img, K.learning_phase()], [saliency])
+
+def modify_backprop(model, name):
+    g = tf.get_default_graph()
+    with g.gradient_override_map({'Relu': name}):
+
+        # get layers that have an activation
+        layer_dict = [layer for layer in model.layers[1:]
+                      if hasattr(layer, 'activation')]
+
+        # replace relu activation
+        for layer in layer_dict:
+            if layer.activation == keras.activations.relu:
+                layer.activation = tf.nn.relu
+
+        # re-instanciate a new model
+        new_model = VGG16(weights='imagenet')
+    return new_model
+
+def deprocess_image(x):
+    '''
+    Same normalization as in:
+    https://github.com/fchollet/keras/blob/master/examples/conv_filter_visualization.py
+    '''
+    if np.ndim(x) > 3:
+        x = np.squeeze(x)
+    # normalize tensor: center on 0., ensure std is 0.1
+    x -= x.mean()
+    x /= (x.std() + 1e-5)
+    x *= 0.1
+
+    # clip to [0, 1]
+    x += 0.5
+    x = np.clip(x, 0, 1)
+
+    # convert to RGB array
+    x *= 255
+    if K.image_dim_ordering() == 'th':
+        x = x.transpose((1, 2, 0))
+    x = np.clip(x, 0, 255).astype('uint8')
+    return x
+
+def grad_cam(input_model, image, category_index, layer_name):
     model = Sequential()
     model.add(input_model)
 
@@ -52,22 +110,34 @@ def grad_cam(input_model, image, category_index, layer_name):
 
     cam = cv2.resize(cam, (224, 224))
     cam = np.maximum(cam, 0)
-    cam = cam / np.max(cam)
+    heatmap = cam / np.max(cam)
 
     #Return to BGR [0..255] from the preprocessed image
     image = image[0, :]
     image -= np.min(image)
     image = np.minimum(image, 255)
 
-    cam = cv2.applyColorMap(np.uint8(255*cam), cv2.COLORMAP_JET)
+    cam = cv2.applyColorMap(np.uint8(255*heatmap), cv2.COLORMAP_JET)
     cam = np.float32(cam) + np.float32(image)
     cam = 255 * cam / np.max(cam)
-    return np.uint8(cam)
+    return np.uint8(cam), heatmap
 
 preprocessed_input = load_image(sys.argv[1])
 
 model = VGG16(weights='imagenet')
 
-predicted_class = np.argmax(model.predict(preprocessed_input))
-cam  = grad_cam(model, preprocessed_input, predicted_class, "block5_pool")
-cv2.imwrite("cam.jpg", cam)
+predictions = model.predict(preprocessed_input)
+top_1 = decode_predictions(predictions)[0][0]
+print('Predicted class:')
+print('%s (%s) with probability %.2f' % (top_1[1], top_1[0], top_1[2]))
+
+predicted_class = np.argmax(predictions)
+cam, heatmap = grad_cam(model, preprocessed_input, predicted_class, "block5_conv3")
+cv2.imwrite("gradcam.jpg", cam)
+
+register_gradient()
+guided_model = modify_backprop(model, 'GuidedBackProp')
+saliency_fn = compile_saliency_function(guided_model)
+saliency = saliency_fn([preprocessed_input, 0])
+gradcam = saliency[0] * heatmap[..., np.newaxis]
+cv2.imwrite("guided_gradcam.jpg", deprocess_image(gradcam))