Major features are extended siamese network support, visualizing histograms and correlation matrices built from activations and weights, extended deconv options, added grayscale support, improved optimize_image and find_max_acts, improved user interface, refactored and fixed bugs

.gitignore

@@ -1,4 +1,4 @@
-settings_local.py
-settings.py.templatec
 *.pyc
-*.DS_Store
+.idea
+settings_user.py
+

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "caffe"]
+	path = caffe
+	url = https://github.com/arikpoz/caffe.git

app_base.py

@@ -6,13 +6,17 @@ class BaseApp(object):
     def __init__(self, settings, key_bindings):
         self.debug_level = 0
 
-    def handle_input(self, input_image, panes):
+    def handle_input(self, input_image, input_label, input_filename, panes):
         pass
 
     def handle_key(self, key, panes):
         '''Handle key and return either key (to let someone downstream handle it) or None (if this app handled it)'''
         pass
 
+    def handle_mouse_left_click(self, x, y, flags, param, panes):
+        '''Handle mouse events'''
+        pass
+
     def redraw_needed(self, key, panes):
         '''App should return whether or not its internal state has
         been updated (perhaps in response to handle_key, handle_input,
@@ -28,7 +32,7 @@ def draw_help(self, panes):
         '''Tells the app to draw its help screen in the given pane. No return necessary.'''
         pass
 
-    def start(self):
+    def start(self, live_vis):
         '''Notify app to start, possibly creating any necessary threads'''
         pass
 

bindings.py

@@ -101,19 +101,29 @@ def get_key_help(self, tag):
 
 _.add('zoom_mode', 'z',
        'Cycle zooming through {currently selected unit, backprop results, none}')
-_.add('pattern_mode', 's',
-       'Toggle overlay of preferred input pattern (regularized optimized images)')
+_.add('next_pattern_mode', 's',
+       'Cycle channels overlay (max opt, max input, weights hist, max hist, correlation, off)')
+_.add('prev_pattern_mode', 'S',
+       'Cycle patterns overlay (max opt, max input, weights hist, max hist, correlation, off)')
+_.add('pattern_first_only', '1',
+       'Toggle pattern loading first image only or loading all available images')
 
-_.add('ez_back_mode_loop', 'b',
-       'Cycle through a few common backprop/deconv modes')
+_.add('next_ez_back_mode_loop', 'b',
+       'Cycle through backprop modes (grad, deconv zf, deconv gb, off)')
+_.add('prev_ez_back_mode_loop', 'B',
+       'Cycle through backprop modes (grad, deconv zf, deconv gb, off)')
 _.add('freeze_back_unit', 'd',
        'Freeze the bprop/deconv origin to be the currently selected unit')
 _.add('show_back', 'a',
        'Toggle between showing forward activations and back/deconv diffs')
-_.add('back_mode', 'n',
-       '(expert) Change back mode directly.')
-_.add('back_filt_mode', 'm',
-       '(expert) Change back output filter directly.')
+_.add('next_back_view_option', 'n',
+       'Cycle through backprop view options (raw, gray, norm, normblur, sum>0, histogram)')
+_.add('prev_back_view_option', 'N',
+       'Cycle through backprop view options (raw, gray, norm, normblur, sum>0, histogram)')
+_.add('next_color_map', 'm',
+       'Cycle through colormaps options (grayscale, jet, plasma)')
+_.add('prev_color_map', 'M',
+       'Cycle through colormaps options (grayscale, jet, plasma)')
 
 _.add('boost_gamma', 't',
        'Boost contrast using gamma correction')
@@ -124,4 +134,15 @@ def get_key_help(self, tag):
 _.add('toggle_unit_jpgs', '9',
        'Turn on or off display of loaded jpg visualization')
 
+_.add('siamese_view_mode', 'v',
+        'Cycle between siamese view modes {first image, second image, both images}')
+
+_.add('toggle_maximal_score', 'r',
+        'Toggle showing maximal score overlays {on, off}')
+_.add('next_input_overlay', 'y',
+      'Cycle input overlay {off, over active, over inactive}')
+_.add('prev_input_overlay', 'Y',
+      'Cycle input overlay {off, over active, over inactive}')
+
+
 bindings = _

build_default.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+cd caffe
+# fix issue with caffe build from source in ubuntu, see more details here: https://github.com/BVLC/caffe/issues/2347
+find . -type f -exec sed -i -e 's^"hdf5.h"^"hdf5/serial/hdf5.h"^g' -e 's^"hdf5_hl.h"^"hdf5/serial/hdf5_hl.h"^g' '{}' \;
+
+# if Makefile.config has already existed, then don't overwrite it
+if [ ! -e  Makefile.config ]; then
+    cp Makefile.config.example Makefile.config
+fi
+make all pycaffe
+cd ..
+cp settings_user.py.example settings_user.py

caffe_misc.py

@@ -0,0 +1,265 @@
+#! /usr/bin/env python
+
+import skimage.io
+import numpy as np
+from image_misc import norm01c
+
+
+def shownet(net):
+    '''Print some stats about a net and its activations'''
+
+    print '%-41s%-31s%s' % ('', 'acts', 'act diffs')
+    print '%-45s%-31s%s' % ('', 'params', 'param diffs')
+    for k, v in net.blobs.items():
+        if k in net.params:
+            params = net.params[k]
+            for pp, blob in enumerate(params):
+                if pp == 0:
+                    print '  ', 'P: %-5s'%k,
+                else:
+                    print ' ' * 11,
+                print '%-32s' % repr(blob.data.shape),
+                print '%-30s' % ('(%g, %g)' % (blob.data.min(), blob.data.max())),
+                print '(%g, %g)' % (blob.diff.min(), blob.diff.max())
+        print '%-5s'%k, '%-34s' % repr(v.data.shape),
+        print '%-30s' % ('(%g, %g)' % (v.data.min(), v.data.max())),
+        print '(%g, %g)' % (v.diff.min(), v.diff.max())
+
+
+class RegionComputer(object):
+    '''Computes regions of possible influcence from higher layers to lower layers.'''
+
+    @staticmethod
+    def region_converter(top_slice, filter_width=(1, 1), stride=(1, 1), pad=(0, 0)):
+        '''
+        Works for conv or pool
+
+        vector<int> ConvolutionLayer<Dtype>::JBY_region_of_influence(const vector<int>& slice) {
+        +  CHECK_EQ(slice.size(), 4) << "slice must have length 4 (ii_start, ii_end, jj_start, jj_end)";
+        +  // Crop region to output size
+        +  vector<int> sl = vector<int>(slice);
+        +  sl[0] = max(0, min(height_out_, slice[0]));
+        +  sl[1] = max(0, min(height_out_, slice[1]));
+        +  sl[2] = max(0, min(width_out_, slice[2]));
+        +  sl[3] = max(0, min(width_out_, slice[3]));
+        +  vector<int> roi;
+        +  roi.resize(4);
+        +  roi[0] = sl[0] * stride_h_ - pad_h_;
+        +  roi[1] = (sl[1]-1) * stride_h_ + kernel_h_ - pad_h_;
+        +  roi[2] = sl[2] * stride_w_ - pad_w_;
+        +  roi[3] = (sl[3]-1) * stride_w_ + kernel_w_ - pad_w_;
+        +  return roi;
+        +}
+        '''
+        assert len(top_slice) == 4
+        assert len(filter_width) == 2
+        assert len(stride) == 2
+        assert len(pad) == 2
+
+        # Crop top slice to allowable region
+        top_slice = [ss for ss in top_slice]  # Copy list or array -> list
+
+        bot_slice = [-123] * 4
+
+        bot_slice[0] = top_slice[0] * stride[0] - pad[0]
+        bot_slice[1] = top_slice[1] * stride[0] - pad[0] + filter_width[0] - 1
+        bot_slice[2] = top_slice[2] * stride[1] - pad[1]
+        bot_slice[3] = top_slice[3] * stride[1] - pad[1] + filter_width[1] - 1
+
+        return bot_slice
+
+
+    @staticmethod
+    def merge_regions(region1, region2):
+
+        region1_x_start, region1_x_end, region1_y_start, region1_y_end = region1
+        region2_x_start, region2_x_end, region2_y_start, region2_y_end = region2
+
+        merged_x_start = min(region1_x_start, region2_x_start)
+        merged_x_end = max(region1_x_end, region2_x_end)
+        merged_y_start = min(region1_y_start, region2_y_start)
+        merged_y_end = max(region1_y_end, region2_y_end)
+
+        merged_region = (merged_x_start, merged_x_end, merged_y_start, merged_y_end)
+
+        return merged_region
+
+
+    @staticmethod
+    def convert_region_dag(settings, from_layer, to_layer, region):
+
+        step_region = None
+
+        layer_def = settings._layer_name_to_record[from_layer] if from_layer in settings._layer_name_to_record else None
+
+        # do single step to convert according to from_layer
+        if not layer_def:
+            # fallback to doing nothing
+            step_region = region
+
+        else:
+
+            if layer_def.type in ['Convolution', 'Pooling']:
+                step_region = RegionComputer.region_converter(region, layer_def.filter, layer_def.stride, layer_def.pad)
+
+            else:
+                # fallback to doing nothing
+                step_region = region
+
+        if from_layer == to_layer:
+            return step_region
+
+        # handle the rest
+        total_region = None
+
+        if layer_def is not None:
+            for parent_layer in layer_def.parents:
+
+                # skip inplace layers
+                if len(parent_layer.tops) == 1 and len(parent_layer.bottoms) == 1 and parent_layer.tops[0] == parent_layer.bottoms[0]:
+                    continue
+
+                # calculate convert_region_dag on each one
+                current_region = RegionComputer.convert_region_dag(settings, parent_layer.name, to_layer, step_region)
+
+                # aggregate results
+                if total_region is None:
+                    total_region = current_region
+                else:
+                    total_region = RegionComputer.merge_regions(total_region, current_region)
+
+        if total_region is None:
+            return step_region
+
+        return total_region
+
+
+def save_caffe_image(img, filename, autoscale = True, autoscale_center = None):
+    '''Takes an image in caffe format (01) or (c01, BGR) and saves it to a file'''
+    if len(img.shape) == 2:
+        # upsample grayscale 01 -> 01c
+        img = np.tile(img[:,:,np.newaxis], (1,1,3))
+    else:
+        img = img[::-1].transpose((1,2,0))
+    if autoscale_center is not None:
+        img = norm01c(img, autoscale_center)
+    elif autoscale:
+        img = img.copy()
+        img -= img.min()
+        img *= 1.0 / (img.max() + 1e-10)
+    skimage.io.imsave(filename, img)
+
+
+def layer_name_to_top_name(net, layer_name):
+
+    if net.top_names.has_key(layer_name) and len(net.top_names[layer_name]) >= 1:
+        return net.top_names[layer_name][0]
+
+    else:
+        return None
+
+def get_max_data_extent(net, settings, layer_name, is_spatial):
+    '''Gets the maximum size of the data layer that can influence a unit on layer.'''
+
+    data_size = net.blobs['data'].data.shape[2:4]  # e.g. (227,227) for fc6,fc7,fc8,prop
+
+    if is_spatial:
+        top_name = layer_name_to_top_name(net, layer_name)
+        conv_size = net.blobs[top_name].data.shape[2:4]        # e.g. (13,13) for conv5
+        layer_slice_middle = (conv_size[0]/2,conv_size[0]/2+1, conv_size[1]/2,conv_size[1]/2+1)   # e.g. (6,7,6,7,), the single center unit
+        data_slice = RegionComputer.convert_region_dag(settings, layer_name, 'input', layer_slice_middle)
+        data_slice_size = data_slice[1]-data_slice[0], data_slice[3]-data_slice[2]   # e.g. (163, 163) for conv5
+        # crop data slice size to data size
+        data_slice_size = min(data_slice_size[0], data_size[0]), min(data_slice_size[1], data_size[1])
+        return data_slice_size
+    else:
+        # Whole data region
+        return data_size
+
+
+def compute_data_layer_focus_area(is_spatial, ii, jj, settings, layer_name, size_ii, size_jj, data_size_ii, data_size_jj):
+
+    if is_spatial:
+
+        # Compute the focus area of the data layer
+        layer_indices = (ii, ii + 1, jj, jj + 1)
+
+        data_indices = RegionComputer.convert_region_dag(settings, layer_name, 'input', layer_indices)
+        data_ii_start, data_ii_end, data_jj_start, data_jj_end = data_indices
+
+        # safe guard edges
+        data_ii_start = max(data_ii_start, 0)
+        data_jj_start = max(data_jj_start, 0)
+        data_ii_end = min(data_ii_end, data_size_ii)
+        data_jj_end = min(data_jj_end, data_size_jj)
+
+        touching_imin = (data_ii_start == 0)
+        touching_jmin = (data_jj_start == 0)
+
+        # Compute how much of the data slice falls outside the actual data [0,max] range
+        ii_outside = size_ii - (data_ii_end - data_ii_start)  # possibly 0
+        jj_outside = size_jj - (data_jj_end - data_jj_start)  # possibly 0
+
+        if touching_imin:
+            out_ii_start = ii_outside
+            out_ii_end = size_ii
+        else:
+            out_ii_start = 0
+            out_ii_end = size_ii - ii_outside
+        if touching_jmin:
+            out_jj_start = jj_outside
+            out_jj_end = size_jj
+        else:
+            out_jj_start = 0
+            out_jj_end = size_jj - jj_outside
+
+    else:
+        data_ii_start, out_ii_start, data_jj_start, out_jj_start = 0, 0, 0, 0
+        data_ii_end, out_ii_end, data_jj_end, out_jj_end = size_ii, size_ii, size_jj, size_jj
+
+    return [out_ii_start, out_ii_end, out_jj_start, out_jj_end, data_ii_start, data_ii_end, data_jj_start, data_jj_end]
+
+
+def extract_patch_from_image(data, net, selected_input_index, settings,
+                             data_ii_end, data_ii_start, data_jj_end, data_jj_start,
+                             out_ii_end, out_ii_start, out_jj_end, out_jj_start, size_ii, size_jj):
+    if settings.is_siamese:
+
+        # input is first image so select first 3 channels
+        if selected_input_index == 0:
+
+            out_arr = np.zeros((3, size_ii, size_jj), dtype='float32')
+            out_arr[:, out_ii_start:out_ii_end, out_jj_start:out_jj_end] = data[0:3,
+                                                                           data_ii_start:data_ii_end,
+                                                                           data_jj_start:data_jj_end]
+        # input is second image so select second 3 channels
+        elif selected_input_index == 1:
+            out_arr = np.zeros((3, size_ii, size_jj), dtype='float32')
+            out_arr[:, out_ii_start:out_ii_end, out_jj_start:out_jj_end] = data[3:6,
+                                                                           data_ii_start:data_ii_end,
+                                                                           data_jj_start:data_jj_end]
+        # input is both images so select concatenate data horizontally
+        elif selected_input_index == -1:
+
+            if settings.siamese_input_mode == 'concat_channelwise':
+                out_arr = np.zeros((3, size_ii, size_jj * 2), dtype='float32')
+                out_arr[:, out_ii_start:out_ii_end, (0 + out_jj_start):(0 + out_jj_end)] = data[0:3,
+                                                                                           data_ii_start:data_ii_end,
+                                                                                           data_jj_start:data_jj_end]
+                out_arr[:, out_ii_start:out_ii_end, (size_jj + out_jj_start):(size_jj + out_jj_end)] = data[3:6,
+                                                                                                       data_ii_start:data_ii_end,
+                                                                                                       data_jj_start:data_jj_end]
+            elif settings.siamese_input_mode == 'concat_along_width':
+                out_arr = np.zeros((3, size_ii, size_jj), dtype='float32')
+                out_arr[:, out_ii_start:out_ii_end, out_jj_start:out_jj_end] = data[:,
+                                                                               data_ii_start:data_ii_end,
+                                                                               data_jj_start:data_jj_end]
+
+        else:
+            print "Error: invalid value for selected_input_index (", selected_input_index, ")"
+    else:
+        out_arr = np.zeros((3, size_ii, size_jj), dtype='float32')
+        out_arr[:, out_ii_start:out_ii_end, out_jj_start:out_jj_end] = data[:,
+                                                                       data_ii_start:data_ii_end,
+                                                                       data_jj_start:data_jj_end]
+    return out_arr