From a1f709e9087d599e9121af94ee95c9fcac82aa4a Mon Sep 17 00:00:00 2001 From: Arijit Kar Date: Tue, 10 Jul 2018 15:05:24 +0530 Subject: [PATCH] Adds boosting tracker --- src/ImageTracking.jl | 8 +- src/boosting_tracker/boosting_tracker.jl | 154 ++++++++++ src/boosting_tracker/online_boosting.jl | 373 +++++++++++++++++++++++ src/core.jl | 18 ++ src/tracker.jl | 33 ++ src/tracker_features.jl | 155 ++++++++++ src/tracker_model.jl | 72 +++++ src/tracker_sampler.jl | 94 ++++++ src/tracker_state_estimator.jl | 104 +++++++ 9 files changed, 1010 insertions(+), 1 deletion(-) create mode 100644 src/boosting_tracker/boosting_tracker.jl create mode 100644 src/boosting_tracker/online_boosting.jl create mode 100644 src/core.jl create mode 100644 src/tracker.jl create mode 100644 src/tracker_features.jl create mode 100644 src/tracker_model.jl create mode 100644 src/tracker_sampler.jl create mode 100644 src/tracker_state_estimator.jl diff --git a/src/ImageTracking.jl b/src/ImageTracking.jl index 1cda465..365d1cd 100644 --- a/src/ImageTracking.jl +++ b/src/ImageTracking.jl @@ -8,13 +8,19 @@ using Interpolations using StaticArrays include("optical_flow.jl") +include("tracker.jl") export # main functions optical_flow, + init_tracker, + update_tracker, # optical flow algorithms - LK + LK, + + #tracking algorithms + TrackerBoosting end diff --git a/src/boosting_tracker/boosting_tracker.jl b/src/boosting_tracker/boosting_tracker.jl new file mode 100644 index 0000000..dd8cb62 --- /dev/null +++ b/src/boosting_tracker/boosting_tracker.jl @@ -0,0 +1,154 @@ +mutable struct TrackerBoosting{I <: Int, F <: Float64} <: Tracker + #Initialized + num_of_classifiers::I + sampler_overlap::F + sampler_search_factor::F + initial_iterations::I + num_of_features::I + + #Uninitialized + haar_tracker_features::HaarTrackerFeatures + sampler::TrackerSampler + model::TrackerModel + + TrackerBoosting(num_of_classifiers::I = 100, sampler_overlap::F = 0.99, sampler_search_factor::F = 1.8, initial_iterations::I = 20, num_of_features::I = 1050) where {I <: Int, F <: Float64} = new{I, F}( + num_of_classifiers, sampler_overlap, sampler_search_factor, initial_iterations, num_of_features) +end + +function init_impl(tracker::TrackerBoosting, image::Array{T, 2}, bounding_box::MVector{4, Int}) where T + int_image = integral_image(Gray.(image)) + + #Sampling + tracker.sampler = TS_Current_Sample(tracker.sampler_overlap, tracker.sampler_search_factor, :positive) + positive_samples = sample_impl(tracker.sampler, int_image, bounding_box) + + tracker.sampler.mode = :negative + negative_samples = sample_impl(tracker.sampler, int_image, bounding_box) + + if isempty(positive_samples) || isempty(negative_samples) + throw(ArgumentError("Could not get initial samples.")) + end + + ROI = tracker.sampler.sampling_ROI + + #Compute Haar Features + tracker.haar_tracker_features = HaarTrackerFeatures(tracker.num_of_features, MVector{2}((bounding_box[3] - bounding_box[1] + 1), (bounding_box[4] - bounding_box[2] + 1))) + generate_features(tracker.haar_tracker_features, tracker.num_of_features) + + extraction(tracker.haar_tracker_features, MVector{length(positive_samples)}(positive_samples)) + pos_responses = tracker.haar_tracker_features.responses + extraction(tracker.haar_tracker_features, MVector{length(negative_samples)}(negative_samples)) + neg_responses = tracker.haar_tracker_features.responses + + #Model + tracker.model = initialize_boosting_model(bounding_box) + state_estimator = TSE_Adaboost(tracker.num_of_classifiers, tracker.initial_iterations, tracker.num_of_features, false, MVector{2}((bounding_box[3] - bounding_box[1] + 1), (bounding_box[4] - bounding_box[2] + 1)), ROI) + tracker.model.state_estimator = state_estimator + + #Run model estimation and update for initial_iterations iterations + for i = 1:tracker.initial_iterations + #Compute temp features + temp_haar_features = HaarTrackerFeatures(length(positive_samples)+length(negative_samples), MVector{2}((bounding_box[3] - bounding_box[1] + 1), (bounding_box[4] - bounding_box[2] + 1))) + generate_features(temp_haar_features, temp_haar_features.num_of_features) + + #Model estimate + tracker.model.mode = :negative + tracker.model.current_sample = MVector{length(negative_samples)}(negative_samples) + model_estimation(tracker.model, neg_responses, true) + + tracker.model.mode = :positive + tracker.model.current_sample = MVector{length(positive_samples)}(positive_samples) + model_estimation(tracker.model, pos_responses, true) + + #Model update + model_update(tracker.model) + + #Get replaced classifier and change the features + replaced_classifier = tracker.model.state_estimator.replaced_classifier + swapped_classifier = tracker.model.state_estimator.swapped_classifier + + for j = 1:length(replaced_classifier) + if replaced_classifier[j] > 0 && swapped_classifier[j] > 0 + swap_feature(tracker.haar_tracker_features, replaced_classifier[j], swapped_classifier[j]) + swap_feature(tracker.haar_tracker_features, swapped_classifier[j], temp_haar_features.features[j]) + end + end + end +end + +function update_impl(tracker::TrackerBoosting, image::Array{T, 2}) where T + int_image = integral_image(Gray.(image)) + + #Get the last location + last_target_state = tracker.model.trajectory[end] + last_target_bounding_box = MVector{4}(round(Int, last_target_state.position[1]), round(Int, last_target_state.position[2]), round(Int, last_target_state.position[1])+last_target_state.height, round(Int, last_target_state.position[2])+last_target_state.width) + + #Sampling new frame based on last location + tracker.sampler.mode = :classify + detection_samples = sample_impl(tracker.sampler, int_image, last_target_bounding_box) + ROI = tracker.sampler.sampling_ROI + + if isempty(detection_samples) + throw(ArgumentError("Could not get detection samples.")) + end + + classifiers = get_selected_weak_classifier(tracker.model.state_estimator.boost_classifier) + extractor = tracker.haar_tracker_features + responses = extract_selected(extractor, MVector{length(classifiers)}(classifiers), MVector{length(detection_samples)}(detection_samples)) + + #Predict new location + tracker.model.mode = :classify + tracker.model.current_sample = MVector{length(detection_samples)}(detection_samples) + tracker.model.state_estimator.current_confidence_map = model_estimation(tracker.model, responses, false) + tracker.model.state_estimator.sample_ROI = ROI + + run_state_estimator(tracker.model) + current_state = tracker.model.trajectory[end] + new_bounding_box = MVector{4}(round(Int, current_state.position[1]), round(Int, current_state.position[2]), round(Int, current_state.position[1])+current_state.height, round(Int, current_state.position[2])+current_state.width) + + #Sampling new frame based on new location + tracker.sampler.mode = :positive + positive_samples = sample_impl(tracker.sampler, int_image, new_bounding_box) + + tracker.sampler.mode = :negative + negative_samples = sample_impl(tracker.sampler, int_image, new_bounding_box) + + if isempty(positive_samples) || isempty(negative_samples) + throw(ArgumentError("Could not get initial samples.")) + end + + #Extract features + extraction(tracker.haar_tracker_features, MVector{length(positive_samples)}(positive_samples)) + pos_responses = tracker.haar_tracker_features.responses + extraction(tracker.haar_tracker_features, MVector{length(negative_samples)}(negative_samples)) + neg_responses = tracker.haar_tracker_features.responses + + #Compute temp features + temp_haar_features = HaarTrackerFeatures(length(positive_samples)+length(negative_samples), MVector{2}((new_bounding_box[3] - new_bounding_box[1] + 1), (new_bounding_box[4] - new_bounding_box[2] + 1))) + generate_features(temp_haar_features, temp_haar_features.num_of_features) + + #Model estimate + tracker.model.mode = :negative + tracker.model.current_sample = MVector{length(negative_samples)}(negative_samples) + model_estimation(tracker.model, neg_responses, true) + + tracker.model.mode = :positive + tracker.model.current_sample = MVector{length(positive_samples)}(positive_samples) + model_estimation(tracker.model, pos_responses, true) + + #Model update + model_update(tracker.model) + + #Get replaced classifier and change the features + replaced_classifier = tracker.model.state_estimator.replaced_classifier + swapped_classifier = tracker.model.state_estimator.swapped_classifier + + for j = 1:length(replaced_classifier) + if replaced_classifier[j] > 0 && swapped_classifier[j] > 0 + swap_feature(tracker.haar_tracker_features, replaced_classifier[j], swapped_classifier[j]) + swap_feature(tracker.haar_tracker_features, swapped_classifier[j], temp_haar_features.features[j]) + end + end + + return new_bounding_box +end diff --git a/src/boosting_tracker/online_boosting.jl b/src/boosting_tracker/online_boosting.jl new file mode 100644 index 0000000..b9ffa32 --- /dev/null +++ b/src/boosting_tracker/online_boosting.jl @@ -0,0 +1,373 @@ +mutable struct WeakClassifier + μ::Float64 + σ::Float64 + positive_samples_dist::MVector{6, Float64} + negative_samples_dist::MVector{6, Float64} + threshold::Float64 + parity::Int +end + +mutable struct BaseClassifier + weak_classifiers::Vector{WeakClassifier} + reference_weak_classifier::Bool + num_of_weak_classifiers::Int + selected_classifier::Int + new_weak_classifier_index::Int + w_correct::MVector{N, Float64} where N + w_wrong::MVector{N, Float64} where N + initial_iterations::Int +end + +mutable struct Detector + #Initialized + size_of_confidences::Int + num_of_detections::Int + size_of_detections::Int + best_detection_index::Int + max_confidence::Float64 + + #Uinitialized + confidences::MVector{N, Float64} where N + detection_indices::MVector{N, Int} where N + confidence_matrix::MMatrix{N1, N2, Float64} where N1 where N2 + + Detector(size_of_confidences::Int = 0, num_of_detections::Int = 0, size_of_detections::Int = 0, best_detection_index::Int = 0, max_confidence::Float64 = -Inf) = new( + size_of_confidences, num_of_detections, size_of_detections, best_detection_index, max_confidence) +end + +mutable struct StrongClassifier + #Initialized + num_of_base_classifiers::Int + num_of_weak_classifiers::Int + initial_iterations::Int + α::MVector{N, Float64} where N + patch_size::MVector{2, Int} + feature_exchange::Bool + error_mask::MVector{N, Bool} where N + errors::MVector{N, Float64} where N + sum_of_errors::MVector{N, Float64} where N + detector::Detector + ROI::MVector{4, Int} + + #Uinitialized + base_classifiers::MVector{N, BaseClassifier} where N + replaced_classifier::Int + swapped_classifier::Int + + StrongClassifier(num_of_base_classifiers::Int, num_of_weak_classifiers::Int, initial_iterations::Int, α::MVector{N1, Float64}, patch_size::MVector{2, Int}, feature_exchange::Bool, error_mask::MVector{N2, Bool}, errors::MVector{N2, Float64}, sum_of_errors::MVector{N2, Float64}, detector::Detector, ROI::MVector{4, Int}) where N1 where N2 = new( + num_of_base_classifiers, num_of_weak_classifiers, initial_iterations, α, patch_size, feature_exchange, error_mask, errors, sum_of_errors, detector, ROI) +end + +#StrongClassifier + +function initialize_strong_classifier(num_of_base_classifiers::Int, num_of_weak_classifiers::Int, initial_iterations::Int, patch_size::MVector{2, Int}, feature_exchange::Bool, ROI::MVector{4, Int}) + α = MVector{num_of_base_classifiers}(zeros(num_of_base_classifiers,)) + error_mask = MVector{num_of_weak_classifiers + initial_iterations, Bool}() + errors = MVector{num_of_weak_classifiers + initial_iterations, Float64}() + sum_of_errors = MVector{num_of_weak_classifiers + initial_iterations, Float64}() + detector = Detector() + StrongClassifier(num_of_base_classifiers, num_of_weak_classifiers, initial_iterations, α, patch_size, feature_exchange, error_mask, errors, sum_of_errors, detector, ROI) +end + +function init_base_classifiers(sc::StrongClassifier) + tempBaseClassifier = [initialize_base_classifier(sc.num_of_weak_classifiers, sc.initial_iterations)] + for i = 2:sc.num_of_base_classifiers + push!(tempBaseClassifier, initialize_base_classifier(sc.num_of_weak_classifiers, sc.initial_iterations, tempBaseClassifier[1].weak_classifiers)) + end + sc.base_classifiers = MVector{sc.num_of_base_classifiers}(tempBaseClassifier) +end + +function classify(sc::StrongClassifier, images::MVector{N, Array{T, 2}}, sample_ROI::MVector{4, Int}) where T where N + sc.ROI = sample_ROI + index = 0 + confidences = 0.0 + + num_patches = length(images) + sc.detector.size_of_confidences = num_patches + sc.detector.confidences = MVector{num_patches, Float64}() + + sc.detector.num_of_detections = 0 + sc.detector.best_detection_index = -1 + sc.detector.max_confidence = -Inf + + step_row = max(1, floor(Int, 0.01*sc.patch_size[1] + 0.5)) + step_col = max(1, floor(Int, 0.01*sc.patch_size[2] + 0.5)) + + height = floor(Int,(sc.ROI[3] - sc.ROI[1] - sc.patch_size[1])/step_row) + width = floor(Int,(sc.ROI[4] - sc.ROI[2] - sc.patch_size[2])/step_col) + + sc.detector.confidence_matrix = MMatrix{height, width, Float64}() + + current_patch = 1 + for i = 1:width + for j = 1:height + sc.detector.confidences[current_patch] = evaluate(sc, images[current_patch]) + + sc.detector.confidence_matrix[j,i] = sc.detector.confidences[current_patch] + current_patch += 1 + end + end + + sc.detector.confidence_matrix = ImageFiltering.imfilter(sc.detector.confidence_matrix, centered(Kernel.gaussian(0.5)[-1:1,-1:1])) + + min_val = minimum(sc.detector.confidence_matrix) + max_val = maximum(sc.detector.confidence_matrix) + + current_patch = 1 + for i = 1:width + for j = 1:height + sc.detector.confidences[current_patch] = sc.detector.confidence_matrix[j,i] + + if sc.detector.confidences[current_patch] > sc.detector.max_confidence + sc.detector.max_confidence = sc.detector.confidences[current_patch] + sc.detector.best_detection_index = current_patch + end + + if sc.detector.confidences[current_patch] > 0.0 + sc.detector.num_of_detections += 1 + end + + current_patch += 1 + end + end + + sc.detector.size_of_detections = sc.detector.num_of_detections + sc.detector.detection_indices = MVector{sc.detector.num_of_detections, Int}() + + current_detection = 1 + for i = 1:num_patches + if sc.detector.confidences[i] > 0.0 + sc.detector.detection_indices[current_detection] = i + current_detection += 1 + end + end + + if sc.detector.num_of_detections <= 0 + confidences = 0 + return index, confidences + end + + index = sc.detector.best_detection_index + confidences = sc.detector.max_confidence + return index, confidences +end + +function update(sc::StrongClassifier, image::AbstractArray{T, 2}, target::Int, importance::Float64 = 1.0) where T + sc.error_mask .= false + sc.errors .= 0.0 + sc.sum_of_errors .= 0.0 + + train_classifier(sc.base_classifiers[1], image, target, importance, sc.error_mask) + for i = 1:sc.num_of_base_classifiers + selected_classifier = select_best_classifier(sc.base_classifiers[i], sc.errors, sc.error_mask, importance) + + if sc.errors[selected_classifier] >= 0.5 + sc.α[i] = 0 + else + sc.α[i] = log((1.0 - sc.errors[selected_classifier])/sc.errors[selected_classifier]) + end + + if sc.error_mask[selected_classifier] + importance *= sqrt((1.0 - sc.errors[selected_classifier])/sc.errors[selected_classifier]) + else + importance *= sqrt(sc.errors[selected_classifier]/(1.0 - sc.errors[selected_classifier])) + end + + for j = i:sc.num_of_weak_classifiers + if sc.errors[j] != Inf && sc.sum_of_errors[j] >= 0 + sc.sum_of_errors[j] += sc.errors[j] + end + end + + sc.sum_of_errors[selected_classifier] = -1 + sc.errors[selected_classifier] = Inf + end + + if sc.feature_exchange + sc.replaced_classifier = compute_replacement(sc.base_classifiers[1], sc.sum_of_errors) + sc.swapped_classifier = sc.base_classifiers[1].new_weak_classifier_index + end + + return true +end + +function replace_weak_classifier(sc::StrongClassifier, index::Int) + if sc.feature_exchange && index > 0 + replace_weak_classifier(sc.base_classifiers[1], index) + + source_index = sc.base_classifiers[1].new_weak_classifier_index + target_index = index + + for i = 2:sc.num_of_base_classifiers + assert(target_index > 0) + assert(target_index != sc.base_classifiers[i].selected_classifier) + assert(target_index <= sc.base_classifiers[i].num_of_weak_classifiers) + + sc.base_classifiers[i].w_wrong[target_index] = sc.base_classifiers[i].w_wrong[source_index] + sc.base_classifiers[i].w_wrong[source_index] = 1.0 + sc.base_classifiers[i].w_correct[target_index] = sc.base_classifiers[i].w_correct[source_index] + sc.base_classifiers[i].w_correct[source_index] = 1.0 + end + end +end + +function get_selected_weak_classifier(sc::StrongClassifier) + return map(i -> i.selected_classifier, sc.base_classifiers) +end + +function evaluate(sc::StrongClassifier, response::AbstractArray{T, 2}) where T + result = 0.0 + for i = 1:sc.num_of_base_classifiers + result += evaluate(sc.base_classifiers[i], response)*sc.α[i] + end + return result +end + +#BaseClassifier + +function initialize_base_classifier(num_of_weak_classifiers::Int, initial_iterations::Int) + weak_classifiers = Vector{WeakClassifier}() + for i = 1:num_of_weak_classifiers+initial_iterations + push!(weak_classifiers, WeakClassifier(0.0, 1.0, MVector{6, Float64}(0.0, 1.0, 1000.0, 0.01, 1000.0, 0.01), MVector{6, Float64}(0.0, 1.0, 1000.0, 0.01, 1000.0, 0.01), 0.0, 0.0)) + end + w_correct = MVector{num_of_weak_classifiers+initial_iterations}(ones(num_of_weak_classifiers+initial_iterations,)) + w_wrong = MVector{num_of_weak_classifiers+initial_iterations}(ones(num_of_weak_classifiers+initial_iterations,)) + return BaseClassifier(weak_classifiers, false, num_of_weak_classifiers, 1, num_of_weak_classifiers, w_correct, w_wrong, initial_iterations) +end + +function initialize_base_classifier(num_of_weak_classifiers::Int, initial_iterations::Int, weak_classifiers::Vector{WeakClassifier}) + w_correct = MVector{num_of_weak_classifiers+initial_iterations}(ones(num_of_weak_classifiers+initial_iterations,)) + w_wrong = MVector{num_of_weak_classifiers+initial_iterations}(ones(num_of_weak_classifiers+initial_iterations,)) + return BaseClassifier(weak_classifiers, true, num_of_weak_classifiers, 1, num_of_weak_classifiers, w_correct, w_wrong, initial_iterations) +end + +function evaluate(bc::BaseClassifier, image::AbstractArray{T, 2}) where T + #CHECK: Made it 1 from : in last argument + wc = bc.weak_classifiers[bc.selected_classifier] + value = image[bc.selected_classifier,1] + return (wc.parity*(value - wc.threshold) > 0 ? 1 : -1) +end + +function train_classifier(bc::BaseClassifier, image::AbstractArray{T, 2}, target::Int, importance::Float64, error_mask::MVector{N, Bool}) where T where N + A = 1.0 + K = 0 + K_Max = 10 + while true + A *= rand() + if K > K_Max || A < exp(-importance) + break + end + K += 1 + end + + for i = 1:K+1 + for j = 1:bc.num_of_weak_classifiers+bc.initial_iterations + error_mask[j] = update(bc.weak_classifiers[j], Float64.(image[j,1]), target) + end + end +end + +function select_best_classifier(bc::BaseClassifier, errors::MVector{N, Float64}, errorMask::MVector{N, Bool}, importance::Float64) where N + minError = Inf + temp_selected_classifier = bc.selected_classifier + for i = 1:bc.num_of_weak_classifiers+bc.initial_iterations + if errorMask[i] + bc.w_wrong[i] += importance + else + bc.w_correct[i] += importance + end + + if errors[i] == Inf + continue + end + + errors[i] = bc.w_wrong[i]/(bc.w_correct[i] + bc.w_wrong[i]) + + if i < bc.num_of_weak_classifiers && errors[i] < minError + minError = errors[i] + temp_selected_classifier = i + end + end + + bc.selected_classifier = temp_selected_classifier + return bc.selected_classifier +end + +function replace_weak_classifier(bc::BaseClassifier, index::Int) + bc.weak_classifiers[index] = bc.weak_classifiers[bc.new_weak_classifier_index] + bc.w_wrong[index] = bc.w_wrong[bc.new_weak_classifier_index] + bc.w_correct[index] = bc.w_correct[bc.new_weak_classifier_index] + + bc.w_wrong[bc.new_weak_classifier_index] = 1 + bc.w_correct[bc.new_weak_classifier_index] = 1 + bc.weak_classifiers[bc.new_weak_classifier_index] = WeakClassifier(0.0, 1.0, MVector{6, Float64}(0.0, 1.0, 1000.0, 0.01, 1000.0, 0.01), MVector{6, Float64}(0.0, 1.0, 1000.0, 0.01, 1000.0, 0.01), 0.0, 0.0) +end + +function compute_replacement(bc::BaseClassifier, errors::MVector{N, Float64}) where N + maxError = 0.0 + #Taken index 0 rather than -1 + index = 0 + + for i = bc.num_of_weak_classifiers:-1:1 + if errors[i] > maxError + maxError = errors[i] + index = i + end + end + + assert(index > 0) + assert(index != bc.selected_classifier) + + bc.new_weak_classifier_index += 1 + if bc.new_weak_classifier_index == bc.num_of_weak_classifiers + bc.initial_iterations + bc.new_weak_classifier_index = bc.num_of_weak_classifiers + end + + if maxError > errors[bc.new_weak_classifier_index] + return index + else + return 0 + end +end + +#WeakClassifier + +function update(wc::WeakClassifier, value::Float64, target::Int) + if target == 1 + update(wc.positive_samples_dist, value) + else + update(wc.negative_samples_dist, value) + end + + wc.threshold = (wc.positive_samples_dist[2] + wc.negative_samples_dist[2])/2 + wc.parity = (wc.positive_samples_dist[2] > wc.negative_samples_dist[2] ? 1 : -1) + + return ((wc.parity*(value - wc.threshold) > 0 ? 1 : -1) != target) +end + +#Misc + +function update(gauss_dist::MVector{6, Float64}, value::Float64) + min_factor = 0.001 + + K = gauss_dist[3]/(gauss_dist[3] + gauss_dist[5]) + if K < min_factor + K = min_factor + end + + gauss_dist[1] = K*value + (1 - K)*gauss_dist[1] + gauss_dist[3] = (gauss_dist[3]*gauss_dist[5])/(gauss_dist[3] + gauss_dist[5]) + + K = gauss_dist[4]/(gauss_dist[4] + gauss_dist[6]) + if K < min_factor + K = min_factor + end + + temp_σ = K*(gauss_dist[1] - value)^2 + (1 - K)*gauss_dist[2]^2 + gauss_dist[4] = (gauss_dist[4]*gauss_dist[6])/(gauss_dist[4] + gauss_dist[6]) + + gauss_dist[2] = sqrt(temp_σ) + if gauss_dist[2] <= 1.0 + gauss_dist[2] = 1.0 + end +end diff --git a/src/core.jl b/src/core.jl new file mode 100644 index 0000000..e5e302f --- /dev/null +++ b/src/core.jl @@ -0,0 +1,18 @@ +mutable struct TrackerTargetState + #Initialized + position::SVector{2, Float64} + height::Int + width::Int + is_target::Bool + + #Uninitialized + responses::Array{T, 2} where T + + TrackerTargetState(position::SVector{2, Float64}, height::Int, width::Int, is_target::Bool) = new( + position, height, width, is_target) +end + +mutable struct ConfidenceMap + states::Vector{TrackerTargetState} + confidence::Vector{Float64} +end diff --git a/src/tracker.jl b/src/tracker.jl new file mode 100644 index 0000000..967c1e8 --- /dev/null +++ b/src/tracker.jl @@ -0,0 +1,33 @@ +abstract type Tracker end + +function init_tracker(tracker::Tracker, image::Array{T, 2}, bounding_box::MVector{4, Int}) where T + @assert bounding_box[1] >= 1 && bounding_box[1] <= size(image)[1] + @assert bounding_box[2] >= 1 && bounding_box[2] <= size(image)[2] + @assert bounding_box[3] >= 1 && bounding_box[3] <= size(image)[1] + @assert bounding_box[4] >= 1 && bounding_box[4] <= size(image)[2] + + @assert bounding_box[1] < bounding_box[3] + @assert bounding_box[2] < bounding_box[4] + + init_impl(tracker, image, bounding_box) +end + +function update_tracker(tracker::Tracker, image::Array{T, 2}) where T + update_impl(tracker, image) +end + +#--------------------- +# TRACKER COMPONENTS +#--------------------- + +include("core.jl") +include("tracker_state_estimator.jl") +include("tracker_model.jl") +include("tracker_sampler.jl") +include("tracker_features.jl") + +#------------------ +# IMPLEMENTATIONS +#------------------ + +include("boosting_tracker/boosting_tracker.jl") diff --git a/src/tracker_features.jl b/src/tracker_features.jl new file mode 100644 index 0000000..7058c5c --- /dev/null +++ b/src/tracker_features.jl @@ -0,0 +1,155 @@ +mutable struct SingleHaarFeature + #Uninitialized + rect_type::Symbol + num_of_areas::Int + areas::MMatrix{4, N, Int} where N + weights::MVector{N, Float64} where N + + SingleHaarFeature() = new() +end + +mutable struct HaarTrackerFeatures{I <: Int} + #Initialized + num_of_features::I + patch_size::MVector{2, I} + + #Uinitialized + features::Vector{SingleHaarFeature} + responses::Array{T, 2} where T + + HaarTrackerFeatures(num_of_features::I, patch_size::MVector{2, I}) where I <: Int = new{I}( + num_of_features, patch_size) +end + +function initialize_feature_haar(patch_size::MVector{2, Int}) + selected = false + + haar_feature = SingleHaarFeature() + + while !selected + position = SVector{2}(round(Int, rand()*(patch_size[1] - 1) + 1), round(Int, rand()*(patch_size[2] - 1) + 1)) + dimensions = SVector{2}(round(Int, 0.75*rand()*patch_size[1]), round(Int, 0.75*rand()*patch_size[2])) + + case = rand([1, 2, 3, 4, 5]) + + if case == 1 + #CHECK: Added -1 here + if position[1] + dimensions[1] - 1 > patch_size[1] || position[2] + 2*dimensions[2] - 1 > patch_size[2] || 2*dimensions[1]*dimensions[2] < 9 + continue + end + + haar_feature.rect_type = :x2 + haar_feature.num_of_areas = 2 + haar_feature.weights = MVector{2}(1.0, -1.0) + haar_feature.areas = MMatrix{4,2}(position[1], position[2], position[1] + dimensions[1] - 1, position[2] + dimensions[2] - 1, + position[1], position[2] + dimensions[2], position[1] + dimensions[1] - 1, position[2] + 2*dimensions[2] - 1) + + selected = true + elseif case == 2 + if position[1] + 2*dimensions[1] - 1 > patch_size[1] || position[2] + dimensions[2] - 1 > patch_size[2] || 2*dimensions[1]*dimensions[2] < 9 + continue + end + + haar_feature.rect_type = :y2 + haar_feature.num_of_areas = 2 + haar_feature.weights = MVector{2}(1.0, -1.0) + haar_feature.areas = MMatrix{4,2}(position[1], position[2], position[1] + dimensions[1] - 1, position[2] + dimensions[2] - 1, + position[1] + dimensions[1], position[2], position[1] + 2*dimensions[1] - 1, position[2] + dimensions[2] - 1) + + selected = true + elseif case == 3 + if position[1] + dimensions[1] - 1 > patch_size[1] || position[2] + 4*dimensions[2] - 1 > patch_size[2] || 4*dimensions[1]*dimensions[2] < 9 + continue + end + + haar_feature.rect_type = :x3 + haar_feature.num_of_areas = 3 + haar_feature.weights = MVector{3}(1.0, -2.0, 1.0) + haar_feature.areas = MMatrix{4,3}(position[1], position[2], position[1] + dimensions[1] - 1, position[2] + dimensions[2] - 1, + position[1], position[2] + dimensions[2], position[1] + dimensions[1] - 1, position[2] + 3*dimensions[2] - 1, + position[1], position[2] + 3*dimensions[2], position[1] + dimensions[1] - 1, position[2] + 4*dimensions[2] - 1) + + selected = true + elseif case == 4 + if position[1] + 4*dimensions[1] - 1 > patch_size[1] || position[2] + dimensions[2] - 1 > patch_size[2] || 4*dimensions[1]*dimensions[2] < 9 + continue + end + + haar_feature.rect_type = :y3 + haar_feature.num_of_areas = 3 + haar_feature.weights = MVector{3}(1.0, -2.0, 1.0) + haar_feature.areas = MMatrix{4,3}(position[1], position[2], position[1] + dimensions[1] - 1, position[2] + dimensions[2] - 1, + position[1] + dimensions[1], position[2], position[1] + 3*dimensions[1] - 1, position[2] + dimensions[2] - 1, + position[1] + 3*dimensions[1], position[2], position[1] + 4*dimensions[1] - 1, position[2] + dimensions[2] - 1) + + selected = true + else + if position[1] + 2*dimensions[1] - 1 > patch_size[1] || position[2] + 2*dimensions[2] - 1 > patch_size[2] || 2*2*dimensions[1]*dimensions[2] < 9 + continue + end + + haar_feature.rect_type = :xy4 + haar_feature.num_of_areas = 3 + haar_feature.weights = MVector{4}(1.0, -1.0, -1.0, 1.0) + haar_feature.areas = MMatrix{4,4}(position[1], position[2], position[1] + dimensions[1] - 1, position[2] + dimensions[2] - 1, + position[1], position[2] + dimensions[2], position[1] + dimensions[1] - 1, position[2] + 2*dimensions[2] - 1, + position[1] + dimensions[1], position[2], position[1] + 2*dimensions[1] - 1, position[2] + dimensions[2] - 1, + position[1] + dimensions[1], position[2] + dimensions[2], position[1] + 2*dimensions[1] - 1, position[2] + 2*dimensions[2] - 1) + + selected = true + end + end + + return haar_feature +end + +function evaluate_haar_feature(feature::SingleHaarFeature, image::Array{T, 2}) where T + result = 0.0 + for i = 1:feature.num_of_areas + result += feature.weights[i]*boxdiff(Float64.(image), feature.areas[1,i]:feature.areas[3,i], feature.areas[2,i]:feature.areas[4,i]) + end + return result +end + +#HaarTrackerFeatures + +function generate_features(features::HaarTrackerFeatures{}, num_of_features::Int) + features.features = Vector{SingleHaarFeature}() + for i = 1:num_of_features + push!(features.features, initialize_feature_haar(features.patch_size)) + end +end + +function swap_feature(features::HaarTrackerFeatures{}, index::Int, feature::SingleHaarFeature) + features.features[index] = feature +end + +function swap_feature(features::HaarTrackerFeatures{}, source_index::Int, target_index::Int) + temp_feature = features.features[source_index] + features.features[source_index] = features.features[target_index] + features.features[target_index] = temp_feature +end + +function extract_selected(features::HaarTrackerFeatures{}, sel_features::MVector{N1, Int}, samples::MVector{N2, Tuple{Array{T, 2}, MVector{2, Int}}}) where N1 where N2 where T + response = Array{Float64}(length(samples), features.num_of_features) + + for i = 1:length(samples) + for j = 1:length(sel_features) + response[i, sel_features[j]] = Float64(evaluate_haar_feature(features.features[sel_features[j]], samples[i][1])) + end + end + + return response +end + +function extraction(features::HaarTrackerFeatures{}, samples::MVector{N, Tuple{Array{T, 2}, MVector{2, Int}}}) where N where T + response = Array{Float64}(length(samples), features.num_of_features) + + for j = 1:features.num_of_features + for i = length(samples) + response[i,j] = evaluate_haar_feature(features.features[j], samples[i][1]) + end + end + + features.responses = response +end diff --git a/src/tracker_model.jl b/src/tracker_model.jl new file mode 100644 index 0000000..c52bf5d --- /dev/null +++ b/src/tracker_model.jl @@ -0,0 +1,72 @@ +abstract type TrackerModel end + +#----------------------------- +# MODEL FOR BOOSTING TRACKER +#----------------------------- + +mutable struct BoostingModel{I <: Int, S <: Symbol} <: TrackerModel + #Initialized + max_length_cmaps_vector::I + mode::S + trajectory::Vector{TrackerTargetState} + confidence_maps::Vector{ConfidenceMap} + + #Uninitialized + state_estimator::TrackerStateEstimator + current_confidence_map::ConfidenceMap + current_sample::MVector{N, Tuple{Array{T, 2}, MVector{2, I}}} where N where T + + BoostingModel(max_length_cmaps_vector::I, mode::S, trajectory::Vector{TrackerTargetState}) where {I <: Int, S <: Symbol} = new{I, S}(max_length_cmaps_vector, mode, trajectory, Vector{ConfidenceMap}()) +end + +function initialize_boosting_model(bounding_box::MVector{4, Int}, max_length_cmaps_vector::Int = 10) + init_state = TrackerTargetState(SVector{2, Float64}(Float64(bounding_box[1]), Float64(bounding_box[2])), round(Int, bounding_box[3] - bounding_box[1]), round(Int, bounding_box[4] - bounding_box[2]), true) + trajectory = [init_state] + return BoostingModel(10, :positive, trajectory) +end + +function model_estimation(model::BoostingModel, responses::Array{T, 2}, add_to_model::Bool) where T + if isempty(model.current_sample) + throw(ArgumentError("The samples in model estimation are empty.")) + end + + confidence_map = ConfidenceMap(Vector{TrackerTargetState}(), Vector{Float64}()) + for i = 1:length(model.current_sample) + if model.mode == :positive || model.mode == :classify + foreground = true + else + foreground = false + end + + current_state = TrackerTargetState(Float64.(model.current_sample[i][2][:]), size(model.current_sample[i][1])[1], size(model.current_sample[i][1])[2], foreground) + temp_resp = responses[i,:] + current_state.responses = reshape(temp_resp, length(temp_resp), 1) + + push!(confidence_map.states, current_state) + push!(confidence_map.confidence, 0.0) + end + + if add_to_model + model.current_confidence_map = confidence_map + end + + return confidence_map +end + +function model_update(model::BoostingModel) + if model.max_length_cmaps_vector != -1 && length(model.confidence_maps) >= model.max_length_cmaps_vector-1 + model.confidence_maps = model.confidence_maps[floor(Int, model.max_length_cmaps_vector/2) + 1:end] + end + + if model.max_length_cmaps_vector != -1 && length(model.trajectory) >= model.max_length_cmaps_vector-1 + model.trajectory = model.trajectory[floor(Int, model.max_length_cmaps_vector/2) + 1:end] + end + + push!(model.confidence_maps, model.current_confidence_map) + update(model.state_estimator, SVector{length(model.confidence_maps)}(model.confidence_maps)) +end + +function run_state_estimator(model::BoostingModel) + target_state = estimate(model.state_estimator, SVector{length(model.confidence_maps)}(model.confidence_maps)) + push!(model.trajectory, target_state) +end diff --git a/src/tracker_sampler.jl b/src/tracker_sampler.jl new file mode 100644 index 0000000..386a99e --- /dev/null +++ b/src/tracker_sampler.jl @@ -0,0 +1,94 @@ +abstract type TrackerSampler end + +function sample(sampler::TrackerSampler, image::Array{T, 2}, ROI::MVector{4, Int}) where T + sample_impl(sampler, image, ROI) +end + +#----------------------------------- +# CURRENT STATE SAMPLER ALGORITHM +# Used by Boosting Tracker +#----------------------------------- + +mutable struct TS_Current_Sample{F <: Float64, S <: Symbol} <: TrackerSampler + overlap::F + search_factor::F + mode::S + sampling_ROI::MVector{4, Int} + valid_ROI::MVector{4, Int} + tracked_patch::MVector{4, Int} +end + +TS_Current_Sample(overlap::F = 0.99, search_factor::F = 1.8, mode::S = :positive) where {F <: Float64, S <: Symbol} = TS_Current_Sample{F, S}( + overlap, search_factor, mode, MVector{4, Int}(),MVector{4, Int}(), MVector{4, Int}()) + +function sample_impl(sampler::TS_Current_Sample{}, image::Array{T, 2}, ROI::MVector{4, Int}) where T + sampler.tracked_patch = ROI + sampler.valid_ROI = MVector{4}(1, 1, size(image)[1], size(image)[2]) + + height = ROI[3] - ROI[1] + 1 + width = ROI[4] - ROI[2] + 1 + + tracking_ROI = MVector{4, Int}() + tracking_ROI[1] = max(0, floor(Int, ROI[1] - height*((sampler.search_factor - 1)/2)) + 1) + tracking_ROI[2] = max(0, floor(Int, ROI[2] - width*((sampler.search_factor - 1)/2)) + 1) + tracking_ROI_height = min(floor(Int, height*sampler.search_factor), size(image)[1] - tracking_ROI[1] + 1) + tracking_ROI_width = min(floor(Int, width*sampler.search_factor), size(image)[2] - tracking_ROI[2] + 1) + + tracking_ROI[3] = tracking_ROI[1] + tracking_ROI_height - 1 + tracking_ROI[4] = tracking_ROI[2] + tracking_ROI_width - 1 + + return sampling(sampler, image, tracking_ROI) +end + +function sampling(sampler::TS_Current_Sample{}, image::Array{T, 2}, tracking_ROI::MVector{4, Int}) where T + sample = Array{Tuple{Array{T, 2}, MVector{2, Int}}, 1}() + + if sampler.valid_ROI == tracking_ROI + sampler.sampling_ROI = tracking_ROI + else + sampler.sampling_ROI[1] = (tracking_ROI[1] < sampler.valid_ROI[1]) ? sampler.valid_ROI[1] : tracking_ROI[1] + sampler.sampling_ROI[2] = (tracking_ROI[2] < sampler.valid_ROI[2]) ? sampler.valid_ROI[2] : tracking_ROI[2] + sampler.sampling_ROI[3] = (tracking_ROI[3] > sampler.valid_ROI[3]) ? sampler.valid_ROI[3] : tracking_ROI[3] + sampler.sampling_ROI[4] = (tracking_ROI[4] > sampler.valid_ROI[4]) ? sampler.valid_ROI[4] : tracking_ROI[4] + end + + if sampler.mode == :positive + positive_sample = image[sampler.tracked_patch[1]:sampler.tracked_patch[3], sampler.tracked_patch[2]:sampler.tracked_patch[4]] + positive_sample_pos = MVector{2}(sampler.tracked_patch[1], sampler.tracked_patch[2]) + for i = 1:4 + push!(sample, (positive_sample, positive_sample_pos)) + end + + return sample + end + + height = sampler.tracked_patch[3] - sampler.tracked_patch[1] + 1 + width = sampler.tracked_patch[4] - sampler.tracked_patch[2] + 1 + + upper_left_block = MVector{4, Int}(sampler.valid_ROI[1], sampler.valid_ROI[2], sampler.valid_ROI[1] + height - 1, sampler.valid_ROI[2] + width - 1) + upper_right_block = MVector{4, Int}(sampler.valid_ROI[1], sampler.valid_ROI[4] - width + 1, sampler.valid_ROI[1] + height - 1, sampler.valid_ROI[4]) + lower_left_block = MVector{4, Int}(sampler.valid_ROI[3] - height + 1, sampler.valid_ROI[2], sampler.valid_ROI[3], sampler.valid_ROI[2] + width - 1) + lower_right_block = MVector{4, Int}(sampler.valid_ROI[3] - height + 1, sampler.valid_ROI[4] - width + 1, sampler.valid_ROI[3], sampler.valid_ROI[4]) + + if sampler.mode == :negative + push!(sample, (image[upper_left_block[1]:upper_left_block[3], upper_left_block[2]:upper_left_block[4]], MVector{2}(upper_left_block[1], upper_left_block[2]))) + push!(sample, (image[upper_right_block[1]:upper_right_block[3], upper_right_block[2]:upper_right_block[4]], MVector{2}(upper_right_block[1], upper_right_block[2]))) + push!(sample, (image[lower_left_block[1]:lower_left_block[3], lower_left_block[2]:lower_left_block[4]], MVector{2}(lower_left_block[1], lower_left_block[2]))) + push!(sample, (image[lower_right_block[1]:lower_right_block[3], lower_right_block[2]:lower_right_block[4]], MVector{2}(lower_right_block[1], lower_right_block[2]))) + + return sample + end + + step_row = max(1, floor(Int, (1 - sampler.overlap)*height + 0.5)) + step_col = max(1, floor(Int, (1 - sampler.overlap)*width + 0.5)) + max_row = sampler.sampling_ROI[3] - sampler.sampling_ROI[1] - height + max_col = sampler.sampling_ROI[4] - sampler.sampling_ROI[2] - width + + for i = 1:step_col:max_col + for j = 1:step_row:max_row + push!(sample, (image[j+sampler.sampling_ROI[1]:j+sampler.sampling_ROI[1]+height-1, i+sampler.sampling_ROI[2]:i+sampler.sampling_ROI[2]+width-1], MVector{2}(j+sampler.sampling_ROI[1], i+sampler.sampling_ROI[2]))) + end + end + + return sample +end diff --git a/src/tracker_state_estimator.jl b/src/tracker_state_estimator.jl new file mode 100644 index 0000000..2b58a9b --- /dev/null +++ b/src/tracker_state_estimator.jl @@ -0,0 +1,104 @@ +abstract type TrackerStateEstimator end + +#------------------------------------------------- +# STATE ESTIMATOR ALGORITHM FOR BOOSTING TRACKER +#------------------------------------------------- + +include("boosting_tracker/online_boosting.jl") + +mutable struct TSE_Adaboost{I <: Int, B <: Bool} <: TrackerStateEstimator + #Initialized + num_of_base_classifiers::I + initial_iterations::I + num_of_features::I + is_trained::B + init_patch_size::MVector{2, I} + sample_ROI::MVector{4, I} + + #Uninitialized + boost_classifier::StrongClassifier + replaced_classifier::MVector{N, Int} where N + swapped_classifier::MVector{N, Int} where N + current_confidence_map::ConfidenceMap + + TSE_Adaboost(num_of_base_classifiers::I, initial_iterations::I, num_of_features::I, is_trained::B, init_patch_size::MVector{2, I}, sample_ROI::MVector{4, I}) where {I <: Int, B <: Bool} = new{I, B}( + num_of_base_classifiers, initial_iterations, num_of_features, is_trained, init_patch_size, sample_ROI) +end + +function estimate(estimator::TSE_Adaboost{}, confidence_maps::SVector{N, ConfidenceMap}) where N + data_type = typeof(estimator.current_confidence_map.states[].responses[]) + images = Array{Array{data_type, 2}, 1}() + + for i = 1:length(estimator.current_confidence_map.states) + push!(images, estimator.current_confidence_map.states[i].responses) + end + + index, confidence = classify(estimator.boost_classifier, MVector{length(images)}(images), estimator.sample_ROI) + return estimator.current_confidence_map.states[index] +end + +function update(estimator::TSE_Adaboost{}, confidence_maps::SVector{N, ConfidenceMap}) where N + if !estimator.is_trained + num_weak_classifier = 10*estimator.num_of_base_classifiers + + estimator.boost_classifier = initialize_strong_classifier(estimator.num_of_base_classifiers, num_weak_classifier, estimator.initial_iterations, estimator.init_patch_size, true, estimator.sample_ROI) + init_base_classifiers(estimator.boost_classifier) + + estimator.is_trained = true + end + + last_confidence_map = confidence_maps[end] + feature_exchange = estimator.boost_classifier.feature_exchange + + estimator.replaced_classifier = MVector{length(last_confidence_map.states), Int}() + estimator.swapped_classifier = MVector{length(last_confidence_map.states), Int}() + + for i = 1:ceil(Int, length(last_confidence_map.states)/2) + current_target_state = last_confidence_map.states[i] + + current_foreground = 1 + if !current_target_state.is_target + current_foreground = -1 + end + + res = current_target_state.responses + + update(estimator.boost_classifier, res, current_foreground) + + if feature_exchange + estimator.replaced_classifier[i] = estimator.boost_classifier.replaced_classifier + estimator.swapped_classifier[i] = estimator.boost_classifier.swapped_classifier + + if estimator.boost_classifier.replaced_classifier > 0 && estimator.boost_classifier.swapped_classifier > 0 + replace_weak_classifier(estimator.boost_classifier, estimator.replaced_classifier[i]) + else + estimator.replaced_classifier[i] = -1 + estimator.swapped_classifier[i] = -1 + end + end + + map_position = i + round(Int, length(last_confidence_map.states)/2) + current_target_state_2 = last_confidence_map.states[map_position] + + current_foreground = 1 + if !current_target_state_2.is_target + current_foreground = -1 + end + + res_2 = current_target_state_2.responses + + update(estimator.boost_classifier, res_2, current_foreground) + + if feature_exchange + estimator.replaced_classifier[map_position] = estimator.boost_classifier.replaced_classifier + estimator.swapped_classifier[map_position] = estimator.boost_classifier.swapped_classifier + + if estimator.boost_classifier.replaced_classifier >= 0 && estimator.boost_classifier.swapped_classifier >= 0 + replace_weak_classifier(estimator.boost_classifier, estimator.replaced_classifier[map_position]) + else + estimator.replaced_classifier[map_position] = -1 + estimator.swapped_classifier[map_position] = -1 + end + end + end +end