Adds MIL tracker

src/ImageTracking.jl

@@ -14,17 +14,20 @@ include("haar.jl")
 export
 
 	# main functions
-  optical_flow,
+	optical_flow,
+	init_tracker,
+	update_tracker,
 
 	# other functions
+	polynomial_expansion,
 	haar_coordinates,
 	haar_features,
 
-	# other functions
-	polynomial_expansion,
-
 	# optical flow algorithms
 	LK,
-	Farneback
+	Farneback,
+
+	# tracking algorithms
+	TrackerMIL
 
 end

src/core.jl

@@ -30,3 +30,22 @@ function pinv2x2(M::AbstractArray)
      D = SMatrix{2,2,Float64}( S[1,1] > tol ?  1/S[1,1] : 0.0 , 0.0, 0.0, S[2,2] > tol ?  1/S[2,2] : 0.0 )
      U*D*V'
 end
+
+mutable struct TrackerTargetState
+    #Initialized
+    position::SVector{2, Float64}
+    height::Int
+    width::Int
+    is_target::Bool
+
+    #Uninitialized
+    responses::Array{T, 2} where T
+
+    TrackerTargetState(position::SVector{2, Float64}, height::Int, width::Int, is_target::Bool) = new(
+                       position, height, width, is_target)
+end
+
+mutable struct ConfidenceMap
+    states::Vector{TrackerTargetState}
+    confidence::Vector{Float64}
+end

src/mil_tracker/mil_tracker.jl

@@ -0,0 +1,119 @@
+mutable struct TrackerMIL{F <: Float64, I <: Int} <: Tracker
+    #Initialized
+    initial_inner_radius::F
+    window_size::F
+    initial_max_negatives::I
+    tracking_inner_radius::F
+    tracking_max_positives::I
+    tracking_max_negatives::I
+    num_of_features::I
+
+    #Uninitialized
+    haar_tracker_features::HaarTrackerFeatures
+    sampler::TrackerSampler
+    model::TrackerModel
+
+    TrackerMIL(initial_inner_radius::F = 3.0, window_size::F = 25.0,  initial_max_negatives::I = 65, tracking_inner_radius::F = 4.0, tracking_max_positives::I = 100000, tracking_max_negatives::I = 65, num_of_features::I = 250) where {F <: Float64, I <: Int}= new{F, I}(
+               initial_inner_radius, window_size, initial_max_negatives, tracking_inner_radius, tracking_max_positives, tracking_max_negatives, num_of_features)
+end
+
+function init_impl(tracker::TrackerMIL{}, image::Array{T, 2}, bounding_box::MVector{4, Int}) where T
+    srand(1)
+    int_image = integral_image(Gray.(image))
+
+    tracker.sampler = TS_Current_Sample_Centered(tracker.initial_inner_radius, tracker.window_size, tracker.initial_max_negatives, tracker.tracking_inner_radius, tracker.tracking_max_positives, tracker.tracking_max_negatives, :init_positive)
+    #Positive Sampling
+    positive_samples = sample(tracker.sampler, int_image, bounding_box)
+    #Negative Sampling
+    tracker.sampler.mode = :init_negative
+    negative_samples = sample(tracker.sampler, int_image, bounding_box)
+
+    if isempty(positive_samples) || isempty(negative_samples)
+        throw(ArgumentError("Could not get initial samples."))
+    end
+
+    #Compute Haar Features
+    tracker.haar_tracker_features = HaarTrackerFeatures(tracker.num_of_features, MVector{2}((bounding_box[3] - bounding_box[1] + 1), (bounding_box[4] - bounding_box[2] + 1)))
+    generate_features(tracker.haar_tracker_features, tracker.num_of_features)
+
+    extraction(tracker.haar_tracker_features, MVector{length(positive_samples)}(positive_samples))
+    pos_responses = [tracker.haar_tracker_features.responses]
+    extraction(tracker.haar_tracker_features, MVector{length(negative_samples)}(negative_samples))
+    neg_responses = [tracker.haar_tracker_features.responses]
+
+    #Model
+    tracker.model = initialize_mil_model(bounding_box)
+    tracker.model.state_estimator = TSE_MIL(tracker.num_of_features, false)
+
+    #Model estimation and update
+    tracker.model.mode = :positive
+    tracker.model.current_sample = MVector{length(positive_samples)}(positive_samples)
+    model_estimation(tracker.model, MVector{length(pos_responses)}(pos_responses), true)
+
+    tracker.model.mode = :negative
+    tracker.model.current_sample = MVector{length(negative_samples)}(negative_samples)
+    model_estimation(tracker.model, MVector{length(neg_responses)}(neg_responses), true)
+
+    model_update(tracker.model)
+end
+
+function update_impl(tracker::TrackerMIL{}, image::Array{T, 2}) where T
+    int_image = integral_image(Gray.(image))
+
+    #Get last location
+    last_target_state = tracker.model.trajectory[end]
+    #TODO: Make -1 in Boosting as well
+    last_target_bounding_box = MVector{4}(round(Int, last_target_state.position[1]), round(Int, last_target_state.position[2]), round(Int, last_target_state.position[1])+last_target_state.height - 1, round(Int, last_target_state.position[2])+last_target_state.width - 1)
+
+    #Sampling new frame based on last location
+    tracker.sampler.mode = :detect
+    detection_samples = sample(tracker.sampler, int_image, last_target_bounding_box)
+
+    if isempty(detection_samples)
+        throw(ArgumentError("Could not get detection samples."))
+    end
+
+    #Extract features from new samples
+    extraction(tracker.haar_tracker_features, MVector{length(detection_samples)}(detection_samples))
+    responses = [tracker.haar_tracker_features.responses]
+
+    #Predict new location
+    tracker.model.mode = :classify
+    tracker.model.current_sample = MVector{length(detection_samples)}(detection_samples)
+    tracker.model.state_estimator.current_confidence_map = model_estimation(tracker.model, MVector{length(responses)}(responses), false)
+
+    run_state_estimator(tracker.model)
+    current_state = tracker.model.trajectory[end]
+    new_bounding_box = MVector{4}(round(Int, current_state.position[1]), round(Int, current_state.position[2]), round(Int, current_state.position[1])+current_state.height-1, round(Int, current_state.position[2])+current_state.width-1)
+
+    #Sampling new frame based on new location
+    tracker.sampler.mode = :init_positive
+    positive_samples = sample(tracker.sampler, int_image, new_bounding_box)
+
+    tracker.sampler.mode = :init_negative
+    negative_samples = sample(tracker.sampler, int_image, new_bounding_box)
+
+    if isempty(positive_samples) || isempty(negative_samples)
+        throw(ArgumentError("Could not get initial samples."))
+    end
+
+    #Extract features
+    extraction(tracker.haar_tracker_features, MVector{length(positive_samples)}(positive_samples))
+    pos_responses = [tracker.haar_tracker_features.responses]
+    extraction(tracker.haar_tracker_features, MVector{length(negative_samples)}(negative_samples))
+    neg_responses = [tracker.haar_tracker_features.responses]
+
+    #Model estimate
+    tracker.model.mode = :negative
+    tracker.model.current_sample = MVector{length(negative_samples)}(negative_samples)
+    model_estimation(tracker.model, MVector{length(neg_responses)}(neg_responses), true)
+
+    tracker.model.mode = :positive
+    tracker.model.current_sample = MVector{length(positive_samples)}(positive_samples)
+    model_estimation(tracker.model, MVector{length(pos_responses)}(pos_responses), true)
+
+    #Model update
+    model_update(tracker.model)
+
+    return new_bounding_box
+end

src/mil_tracker/online_mil.jl

@@ -0,0 +1,196 @@
+mutable struct Stump{I <: Int, F <: Float64, B <: Bool}
+    #Initialized
+    index::I
+    μ0::F
+    μ1::F
+    σ0::F
+    σ1::F
+    l_rate::F
+    trained::B
+
+    #Uninitialized
+    q::F
+    s::I
+    log_n0::F
+    log_n1::F
+    e0::F
+    e1::F
+
+    Stump(index::I = -1, μ0::F = 0.0, μ1::F = 0.0, σ0::F = 1.0, σ1::F = 1.0, l_rate::F = 0.85, trained::B = false) where {I <: Int, F <: Float64, B <: Bool} = new{I, F, B}(
+          index, μ0, μ1, σ0, σ1, l_rate, trained)
+end
+
+mutable struct MIL_Boost_Classifier{I <: Int, F <: Float64}
+    num_sel::I
+    num_feat::I
+    l_rate::F
+    num_samples::I
+    counter::I
+    weak_clf::MVector{N, Stump} where N
+    selectors::Set{I}
+
+    MIL_Boost_Classifier(num_sel::I, num_feat::I, l_rate::F, num_samples::I = 0, counter::I = 0) where {I <: Int, F <: Float64} = new{I, F}(
+                         num_sel, num_feat, l_rate, num_samples, counter)
+end
+
+#Misc
+
+@inline function sigmoid(x::Real)
+    1/(1 + exp(-x))
+end
+
+#Stump
+
+#TODO: Check the last two parameters
+function update(stump::Stump, posx::Array{T, 2}, negx::Array{T, 2}, posw::Array{Float64, 2}, negw::Array{Float64, 2}) where T
+    if size(posx)[2] > 0
+        pos_μ = mean(posx[:, stump.index])
+    else
+        pos_μ = 0.0
+    end
+    if size(negx)[2] > 0
+        neg_μ = mean(negx[:, stump.index])
+    else
+        neg_μ = 0.0
+    end
+
+    if stump.trained
+        if size(posx)[2] > 0
+            stump.μ1 = (1 - stump.l_rate)*pos_μ + stump.μ1*stump.l_rate
+            diff = posx[:,stump.index] .- μ1
+            stump.σ1 = (1 - stump.l_rate)*mean(diff.*diff) + stump.σ1*stump.l_rate
+        end
+        if size(negx)[2] > 0
+            stump.μ0 = (1 - stump.l_rate)*neg_μ + stump.μ0*stump.l_rate
+            diff = negx[:,stump.index] .- μ0
+            stump.σ0 = (1 - stump.l_rate)*mean(diff.*diff) + stump.σ0*stump.l_rate
+        end
+    else
+        stump.trained = true
+        if size(posx)[2] > 0
+            stump.μ1 = pos_μ
+            σ = std(posx[:,stump.index])
+            stump.σ1 = σ*σ + 10^-9
+        end
+        if size(negx)[2] > 0
+            stump.μ0 = neg_μ
+            σ = std(negx[:,stump.index])
+            stump.σ0 = σ*σ + 10^-9
+        end
+    end
+
+    stump.q = (stump.μ1 - stump.μ0)/2
+    stump.s = sign(stump.μ1 - stump.μ0)
+    stump.log_n0 = log(1/(sqrt(stump.σ0)))
+    stump.log_n1 = log(1/(sqrt(stump.σ1)))
+    stump.e0 = -1/(2*stump.σ0)
+    stump.e1 = -1/(2*stump.σ1)
+end
+
+function classify(stump::Stump, x::Array{T, 2}, i::Int) where T
+    xx = x[stump.index, i]
+    log_p0 = stump.e0*(xx - stump.μ0)^2 + stump.log_n0
+    log_p1 = stump.e1*(xx - stump.μ1)^2 + stump.log_n1
+    return log_p1 > log_p0
+end
+
+function classifyF(stump::Stump, x::Array{T, 2}, i::Int) where T
+    xx = x[stump.index, i]
+    log_p0 = stump.e0*(xx - stump.μ0)^2 + stump.log_n0
+    log_p1 = stump.e1*(xx - stump.μ1)^2 + stump.log_n1
+    return log_p1 - log_p0
+end
+
+function classifySetF(stump::Stump, x::Array{T, 2}) where T
+    res = SharedVector{Float64}(size(x)[1])
+    @parallel for i = 1:length(res)
+        res[i] = classifyF(stump, x, i)
+    end
+    return res
+end
+
+#MIL_Boost_Classifier
+
+function initialize_ClfMilBoost(num_sel::Int = 50, num_feat::Int = 250, l_rate::Float64 = 0.85)
+    boost_classifier = MIL_Boost_Classifier(num_sel, num_feat, l_rate)
+    weak_clf = Vector{Stump}()
+    for i = 1:num_feat
+        push!(weak_clf, Stump(i))
+        weak_clf[i].l_rate = l_rate
+    end
+    boost_classifier.weak_clf = MVector{num_feat, Stump}(weak_clf)
+    boost_classifier.selectors = Set{Int}()
+    return boost_classifier
+end
+
+function update(boost_classifier::MIL_Boost_Classifier, posx::Array{T, 2}, negx::Array{T, 2}) where T
+    Hpos = SharedVector{Float64}(size(posx)[1])
+    Hneg = SharedVector{Float64}(size(negx)[1])
+
+    posw = MVector{size(posx)[1], Float64}()
+    negw = MVector{size(negx)[1], Float64}()
+    pospred = SharedArray{Float64}(length(boost_classifier.weak_clf), size(posx)[1])
+    negpred = SharedArray{Float64}(length(boost_classifier.weak_clf), size(negx)[1])
+
+    @parallel for i = 1:boost_classifier.num_feat
+        update(boost_classifier.weak_clf[i], posx, negx)
+        pospred[i, :] = classifySetF(boost_classifier.weak_clf[i], posx)
+        negpred[i, :] = classifySetF(boost_classifier.weak_clf[i], negx)
+    end
+
+    for i = 1:boost_classifier.num_sel
+        poslikl = SharedVector{Float64}(length(boost_classifier.weak_clf))
+        neglikl = SharedVector{Float64}(length(boost_classifier.weak_clf))
+        likl = SharedVector{Float64}(length(boost_classifier.weak_clf))
+
+        @parallel for j = 1:length(boost_classifier.weak_clf)
+            lll = 1.0
+            for k = 1:size(posx)[1]
+                lll *= (1 - sigmoid(Hpos[k] + pospred[j, k]))
+            end
+            poslikl[j] = -log(1 - lll + 10^-5)
+
+            lll = 0.0
+            for k = 1:size(negx)[1]
+                lll += -log(1 - sigmoid(Hneg[k] + negpred[j, k]) + 10^-5)
+            end
+            neglikl[j] = lll
+
+            likl[j] = (poslikl[j]/size(posx)[1]) + (neglikl[j]/size(negx)[1])
+        end
+
+        order = sortperm(likl)
+        for j = 1:length(order)
+            if in(order[j], boost_classifier.selectors) == 0
+                push!(boost_classifier.selectors, order[j])
+                break
+            end
+        end
+
+        @parallel for j = 1:size(posx)[1]
+            Hpos[j] += pospred[boost_classifier.selectors[i], j]
+        end
+        @parallel for j = 1:size(negx)[1]
+            Hneg[j] += negpred[boost_classifier.selectors[i], j]
+        end
+    end
+
+    boost_classifier.counter += 1
+end
+
+function classify(boost_classifier::MIL_Boost_Classifier, x::Array{T, 2}, log_R::Bool = true) where T
+    res = SharedVector{Float64}(size(x)[1])
+
+    for i in boost_classifier.selectors
+        tr = classifySetF(boost_classifier.weak_clf[i], x)
+        @parallel for j = 1:size(x)[1]
+            res[j] += tr[j]
+        end
+    end
+
+    if !log_R
+        res = sigmoid.(res)
+    end
+
+    return res
+end

src/tracker.jl

@@ -0,0 +1,33 @@
+abstract type Tracker end
+
+function init_tracker(tracker::Tracker, image::Array{T, 2}, bounding_box::MVector{4, Int}) where T
+    @assert bounding_box[1] >= 1 && bounding_box[1] <= size(image)[1]
+    @assert bounding_box[2] >= 1 && bounding_box[2] <= size(image)[2]
+    @assert bounding_box[3] >= 1 && bounding_box[3] <= size(image)[1]
+    @assert bounding_box[4] >= 1 && bounding_box[4] <= size(image)[2]
+
+    @assert bounding_box[1] < bounding_box[3]
+    @assert bounding_box[2] < bounding_box[4]
+
+    init_impl(tracker, image, bounding_box)
+end
+
+function update_tracker(tracker::Tracker, image::Array{T, 2}) where T
+    update_impl(tracker, image)
+end
+
+#---------------------
+# TRACKER COMPONENTS
+#---------------------
+
+include("core.jl")
+include("tracker_state_estimator.jl")
+include("tracker_model.jl")
+include("tracker_sampler.jl")
+include("tracker_features.jl")
+
+#------------------
+# IMPLEMENTATIONS
+#------------------
+
+include("mil_tracker/mil_tracker.jl")