ignore training subject sets for counters and selection (zooniverse#3140)

* ignore training sets for workflow counts

sets marked as training are not included for workflow retired and classification counts

* add non_training_subject_sets relation to workflow resource

the workflow can report which subjects sets are marked for training data

* use new workflow non training sets method

* ingore training sets on subject selection

by default ignore training subejct sets in selection unless the user specifies a trainign subject set on a grouped workflow

* shift the training sets specs to the correct file

* remove relation comment about migration and index

* use the metadata config for spec setup

* clarify variable and comment wording

* avoid hitting the db for spec setup

stub the non training set relation behaviour as it's tested on the workflow resource

* use workflow configuration attr training_set_ids

consolidate on one way to configure the API and selector systems to determine a workflows training sets.

* always load configuration json attr

whitelist workflow configuration attribute as it's used in subject selection to determine training set ids

* filter the set relation lookups for known fk values

* add docs on training selector configuration

* use reject instead of invert select

Author: camallen

apiary.apib

@@ -2129,6 +2129,18 @@ the strategy. There are 2 valid criteria:
 If retirement is left blank Panoptes defaults to the `classification_count`
 strategy with 15 classifications per subject.
 
+A Workflow has a _configuration_ object that is used to store client configuration details.
+Three attributes in this object are reserved: _training_set_ids_, _training_chances_ and _training_default_chance_,
+_training_set_ids_ are used to choose training data from in the subject selectors
+and to tell the counting systems which subjects to ignore as training subjects do not retire.
+_training_default_chance_ is the fallback chance a user will see a training subject and also applies if _training_chances_ are not set
+_training_chances_ are used to determine when to show a training subject and each index reflects a seen subject by the user.
+E.g [ 1, 0.5, 0.1 ] reflects when i have seen 0 subjects (first element) on your workflow i will have a 100% chance of selecting a training subject.
+After seeing 1 subject, I will have a 50% chance, on the second subject i will have a 10% chance, after that the default chance applies.
+
+Normally when setting training subject selection you would set the retirement
+config to "never_retire" and use another system (e.g. Caesar) to handle retirement.
+
 + Request
 
     + Headers

app/counters/workflow_counter.rb

@@ -17,12 +17,16 @@ def sws_query
     SubjectWorkflowStatus
     .where(workflow_id: workflow.id)
     .joins("INNER JOIN set_member_subjects ON set_member_subjects.subject_id = subject_workflow_counts.subject_id")
-    .where(set_member_subjects: { subject_set_id: subject_set_ids })
+    .where(
+      set_member_subjects: {
+        subject_set_id: non_training_subject_set_ids_scope
+      }
+    )
   end
 
   private
 
-  def subject_set_ids
-    workflow.subject_sets.pluck(:id)
+  def non_training_subject_set_ids_scope
+    workflow.non_training_subject_sets.select(:id)
   end
 end

app/models/workflow.rb

@@ -12,6 +12,10 @@ class Workflow < ActiveRecord::Base
   has_many :subject_workflow_statuses, dependent: :destroy
   has_many :subject_sets_workflows, dependent: :destroy
   has_many :subject_sets, through: :subject_sets_workflows
+  has_many :non_training_subject_sets,
+    ->(workflow) { where.not(subject_sets_workflows: { subject_set_id: workflow.training_set_ids }) },
+    through: :subject_sets_workflows,
+    source: :subject_set
   has_many :set_member_subjects, through: :subject_sets
   has_many :subjects, through: :set_member_subjects
   has_many :classifications, dependent: :restrict_with_exception
@@ -38,7 +42,7 @@ class Workflow < ActiveRecord::Base
     'options' => {'count' => 15}
   }.freeze
 
-  JSON_ATTRIBUTES = %w(tasks retirement aggregation configuration strings steps).freeze
+  JSON_ATTRIBUTES = %w(tasks retirement aggregation strings steps).freeze
 
   # Used by HttpCacheable
   scope :private_scope, -> { where(project_id: Project.private_scope) }
@@ -159,4 +163,9 @@ def finished?
   def retirement_config
     RetirementValidator.new(self).validate
   end
+
+  def training_set_ids
+    config_training_set_ids = Array.wrap(configuration.dig("training_set_ids"))
+    config_training_set_ids.reject { |id| id.to_i.zero? }
+  end
 end

lib/subjects/postgresql_selection.rb

@@ -26,10 +26,14 @@ def selection_strategy
 
     def available
       query = Subjects::SetMemberSubjectSelector.new(workflow, user).set_member_subjects
-      if workflow.grouped
-        query = query.where(subject_set_id: opts[:subject_set_id])
-      end
-      query
+      subject_set_ids = if workflow.grouped
+                            # respect the user if they want to select from a training set
+                            opts[:subject_set_id]
+                          else
+                            # default mode: do not select from training sets
+                            workflow.non_training_subject_sets.pluck(:id)
+                          end
+      query.where(subject_set_id: subject_set_ids)
     end
 
     def limit

spec/counters/workflow_counter_spec.rb

@@ -41,6 +41,20 @@
           expect(counter.classifications).to eq(2)
         end
       end
+
+      context "with subject sets marked as training data" do
+        let(:workflow) { create(:workflow_with_subjects, num_sets: 2) }
+        let(:training_set) { workflow.subject_sets.first }
+        let(:real_set) { workflow.subject_sets.last }
+        before do
+          real_set_ar_collection_proxy = SubjectSet.where(id: real_set)
+          allow(workflow).to receive(:non_training_subject_sets).and_return(real_set_ar_collection_proxy)
+        end
+
+        it "should return non training data classification count only" do
+          expect(counter.classifications).to eq(2)
+        end
+      end
     end
   end
 
@@ -74,6 +88,20 @@
         workflow.subject_sets = []
         expect(counter.retired_subjects).to eq(0)
       end
+
+      context "with subject sets marked as training data" do
+        let(:workflow) { create(:workflow_with_subjects, num_sets: 2) }
+        let(:training_set) { workflow.subject_sets.first }
+        let(:real_set) { workflow.subject_sets.last }
+        before do
+          real_set_ar_collection_proxy = SubjectSet.where(id: real_set)
+          allow(workflow).to receive(:non_training_subject_sets).and_return(real_set_ar_collection_proxy)
+        end
+
+        it "should return non training data retired count only" do
+          expect(counter.retired_subjects).to eq(2)
+        end
+      end
     end
   end
 end

spec/lib/subjects/postgresql_selection_spec.rb

@@ -10,20 +10,19 @@ def update_sms_priorities
 
   describe "selection" do
     let(:user) { User.first }
-    let(:workflow) { Workflow.first }
+    let(:workflow) { create(:workflow_with_subject_sets) }
     let(:sms) { SetMemberSubject.all }
     let(:opts) { {} }
     let(:sms_count) { 25 }
+    let(:uploader) { create(:user) }
     subject { Subjects::PostgresqlSelection.new(workflow, user, opts) }
 
     before do
-      uploader = create(:user)
-      created_workflow = create(:workflow_with_subject_sets)
-      create_list(:subject, sms_count, project: created_workflow.project, uploader: uploader).each do |subject|
+      create_list(:subject, sms_count, project: workflow.project, uploader: uploader).each do |subject|
         create(:set_member_subject,
           setup_subject_workflow_statuses: true,
           subject: subject,
-          subject_set: created_workflow.subject_sets.first
+          subject_set: workflow.subject_sets.first
         )
       end
     end
@@ -35,6 +34,29 @@ def update_sms_priorities
             SetMemberSubject.all
           end
         end
+
+        context "with a training set and a real set with data" do
+          let(:sms_count) { 2 }
+          let(:training_set) { workflow.subject_sets.first }
+          let(:real_set) { workflow.subject_sets.last }
+
+          before do
+            workflow.configuration['training_set_ids'] = training_set.id
+            create(:subject, project: workflow.project, uploader: uploader) do |subject|
+              create(:set_member_subject,
+                setup_subject_workflow_statuses: true,
+                subject: subject,
+                subject_set: real_set
+              )
+            end
+          end
+
+          it "should not include training subject sets in the results" do
+            result_ids = subject.select
+            non_training_subject_ids = real_set.subjects.pluck(:id)
+            expect(result_ids).to match_array(non_training_subject_ids)
+          end
+        end
       end
 
       context "grouped selection" do

spec/models/workflow_spec.rb

@@ -39,11 +39,12 @@
 
   describe "::find_without_json_attrs" do
     let(:workflow) { create(:workflow) }
+    let(:whitelist_json_attrs) { %w(configuration) }
     let(:json_attrs) do
       col_information = Workflow.columns_hash.select do |name, col|
         /\Ajson.*/.match?(col.sql_type)
       end
-      col_information.keys
+      col_information.keys - whitelist_json_attrs
     end
 
     it "should load the workflow without the json attributes" do
@@ -422,4 +423,44 @@
       end
     end
   end
+
+  describe "#training_subject_sets" do
+    let(:training_ids) { ["1"] }
+
+    it "should return the data in the config object" do
+
+      workflow.configuration["training_set_ids"] = training_ids
+      expect(workflow.training_set_ids).to match_array(training_ids)
+    end
+
+    it "should sanitize the return values to known integer values" do
+      workflow.configuration["training_set_ids"] = training_ids | ["test"]
+      expect(workflow.training_set_ids).to match_array(training_ids)
+    end
+  end
+
+  describe "#non_training_subject_sets" do
+    let(:workflow) { create(:workflow_with_subject_sets) }
+    let(:training_set) { workflow.subject_sets.first }
+    let(:real_set) { workflow.subject_sets.last }
+
+    it "should only return subjects sets that are not marked as training" do
+      workflow.configuration["training_set_ids"] = [training_set.id]
+      expect(workflow.non_training_subject_sets).to match_array([real_set])
+    end
+
+    it "should always return all real sets with empty training sets config" do
+      workflow.configuration["training_set_ids"] = []
+      expect(workflow.non_training_subject_sets).to match_array(workflow.subject_sets)
+    end
+
+    it "should always return all real sets with an unkonwn set id" do
+      workflow.configuration["training_set_ids"] = "test"
+      expect(workflow.non_training_subject_sets).to match_array(workflow.subject_sets)
+    end
+
+    it "should always return all real sets with no training sets config" do
+      expect(workflow.non_training_subject_sets).to match_array(workflow.subject_sets)
+    end
+  end
 end