diff --git a/backend/management/commands/check_util.py b/backend/management/commands/check_util.py new file mode 100644 index 00000000..ab4e003d --- /dev/null +++ b/backend/management/commands/check_util.py @@ -0,0 +1,63 @@ +from django.core.management.base import BaseCommand, CommandError +from django.db.models import Count, Q + +from backend.models import Project, Annotation, Document + + +def check_repeat_annotation(cmd_object: BaseCommand): + """ + Check the database to highlight any documents that has more than one annotation by a single annotator. This + only includes annotations with status COMPLETED,PENDING and REJECTED. ABORTED and TIMED out annotations are ignored. + """ + + completed_pending_rejected_filter = Q(status=Annotation.COMPLETED) | Q(status=Annotation.PENDING) | Q(status=Annotation.REJECTED) + repeat_count = Count("annotations") + docs = Document.objects.annotate(num_annotations=repeat_count).filter(num_annotations__gt=1) + docs_count = 0 + repeat_count = 0 + for doc in docs: + annotator_id_set = set() + has_repeat = False + for annotation in doc.annotations.filter(completed_pending_rejected_filter): + if annotation.user.pk not in annotator_id_set: + annotator_id_set.add(annotation.user.pk) + else: + has_repeat = True + + if has_repeat: + repeat_count += 1 + cmd_object.stdout.write(f"Document {doc.pk} has multiple annotations from the same annotator (ID {annotation.user.pk})") + for annotation in doc.annotations.all(): + print( + f"\tAnnotation ID {annotation.pk} Annotator ID {annotation.user.pk}, status: {annotation.status} data: {annotation.data}") + + docs_count += 1 + + cmd_object.stdout.write(f"Check completed\n\tNumber of documents checked: {docs_count}.\tNumber of documents with repeats {repeat_count}.") + + + +class Command(BaseCommand): + + help = "Utility for performing various checks for diagnosing issues" + + def add_arguments(self, parser): + parser.add_argument("check_type", type=str, help="Type of check to perform: repeat_annotation - Check for documents that have repeated annotations") + + + def handle(self, *args, **options): + if "check_type" in options: + if "repeat_annotation" == options["check_type"]: + check_repeat_annotation(self) + return + else: + raise CommandError(f"Unknown check type: {options['check_type']}") + + + + + + + + + diff --git a/backend/models.py b/backend/models.py index f4b7d47f..48965e54 100644 --- a/backend/models.py +++ b/backend/models.py @@ -408,10 +408,10 @@ def get_annotator_document_score(self, user, doc_type): for document in test_docs: # Checks answers for all test documents - user_annotations = document.annotations.filter(user_id=user.pk) + user_annotations = document.annotations.filter(user_id=user.pk, status=Annotation.COMPLETED) if user_annotations.count() > 1: # User should not have more than 1 annotation per document - raise Exception(f"User {user.username} has more than 1 annotation in document") + raise Exception(f"User {user.username} has more than 1 completed annotation in document") annotation = user_annotations.first() diff --git a/backend/tests/test_models.py b/backend/tests/test_models.py index 0dfa9bcb..889b9126 100644 --- a/backend/tests/test_models.py +++ b/backend/tests/test_models.py @@ -678,7 +678,7 @@ def test_get_annotator_document_score(self): # All incorrect for doc in self.test_docs: - anno = Annotation.objects.create(user=annotator, document=doc) + anno = Annotation.objects.create(user=annotator, document=doc, status=Annotation.COMPLETED) anno.data = incorrect_data self.assertEqual(0, self.project.get_annotator_document_score(annotator, DocumentType.TEST)) @@ -690,7 +690,7 @@ def test_get_annotator_document_score(self): "label1": doc.data["gold"]["label1"]["value"], "label2": doc.data["gold"]["label2"]["value"], } - anno = Annotation.objects.create(user=annotator2, document=doc) + anno = Annotation.objects.create(user=annotator2, document=doc, status=Annotation.COMPLETED) anno.data = correct_annotation_data self.assertEqual(self.num_test_docs, self.project.get_annotator_document_score(annotator2, DocumentType.TEST)) @@ -705,7 +705,7 @@ def test_get_annotator_document_score(self): "label2": doc.data["gold"]["label2"]["value"], } data = correct_annotation_data if counter < num_correct else incorrect_data - anno = Annotation.objects.create(user=annotator3, document=doc) + anno = Annotation.objects.create(user=annotator3, document=doc, status=Annotation.COMPLETED) anno.data = data counter += 1 diff --git a/backend/tests/test_rpc_endpoints.py b/backend/tests/test_rpc_endpoints.py index 096f3e2f..f05126ec 100644 --- a/backend/tests/test_rpc_endpoints.py +++ b/backend/tests/test_rpc_endpoints.py @@ -20,7 +20,7 @@ get_project_training_documents, get_project_test_documents, project_annotator_allow_annotation, \ annotator_leave_project, login, change_annotation, delete_annotation_change_history, get_annotation_task_with_id, \ set_user_document_format_preference, initialise, is_authenticated, user_delete_personal_information, \ - user_delete_account, admin_delete_user_personal_information, admin_delete_user + user_delete_account, admin_delete_user_personal_information, admin_delete_user, reject_project_annotator from backend.rpcserver import rpc_method from backend.errors import AuthError from backend.tests.test_models import create_each_annotation_status_for_user, TestUserModelDeleteUser @@ -1761,6 +1761,107 @@ def test_delete_annotation_change_history(self): +class TestMovingUsersToDifferentProjects(TestEndpoint): + + def setUp(self) -> None: + + # Create initial projects (x5) with documents (x10) + self.projects = [] + for i in range(5): + project = Project.objects.create(name="Test1", owner=self.get_default_user()) + self.projects.append(project) + for j in range(10): + Document.objects.create(project=project) + + # Create annotator + self.annotator = get_user_model().objects.create(username="annotator") + + def test_moving_annotator_to_different_project(self): + projects = self.projects + annotator = self.annotator + + annotator_request = self.get_request() + annotator_request.user = annotator + add_project_annotator(self.get_loggedin_request(), projects[0].pk, annotator.username) + + # Check annotator has been added to 0th project + project_annotators = get_project_annotators(self.get_loggedin_request(), projects[0].pk) + self.assertTrue(project_annotators[0]["username"] == annotator.username) + + # Make two annotations + self.get_and_complete_annotation_task(annotator_request, projects[0]) + self.get_and_complete_annotation_task(annotator_request, projects[0]) + + # Make a pending annotation + annotation_task = get_annotation_task(annotator_request) + + # Remove annotator from 0th project, add to 1st project + remove_project_annotator(self.get_loggedin_request(), projects[0].pk, annotator.username) + add_project_annotator(self.get_loggedin_request(), projects[1].pk, annotator.username) + + # Make two more annotations + self.get_and_complete_annotation_task(annotator_request, projects[1]) + self.get_and_complete_annotation_task(annotator_request, projects[1]) + + # Remove annotator from 1st project, add to 2nd project + remove_project_annotator(self.get_loggedin_request(), projects[1].pk, annotator.username) + add_project_annotator(self.get_loggedin_request(), projects[2].pk, annotator.username) + + # Make two more annotations + self.get_and_complete_annotation_task(annotator_request, projects[2]) + self.get_and_complete_annotation_task(annotator_request, projects[2]) + + # Check no. of associated annotations + self.assertEqual(7, Annotation.objects.filter(user=annotator).count(), "Annotator should have 7 associated annotations") + + def get_and_complete_annotation_task(self, annotator_request, project): + annotation_task = get_annotation_task(annotator_request) + self.assertEqual(project.pk, annotation_task["project_id"], f"Id of the project should be {project.pk}") + complete_annotation_task(annotator_request, annotation_task["annotation_id"], {"test": "result"}) + + + def test_moving_rejected_annotator_to_different_project(self): + projects = self.projects + annotator = self.annotator + + annotator_request = self.get_request() + annotator_request.user = annotator + add_project_annotator(self.get_loggedin_request(), projects[0].pk, annotator.username) + + # Check annotator has been added to 0th project + project_annotators = get_project_annotators(self.get_loggedin_request(), projects[0].pk) + self.assertTrue(project_annotators[0]["username"] == annotator.username) + + # Make two annotations + self.get_and_complete_annotation_task(annotator_request, projects[0]) + self.get_and_complete_annotation_task(annotator_request, projects[0]) + + # Make a pending annotation + annotation_task = get_annotation_task(annotator_request) + + # Remove annotator from 0th project, add to 1st project + reject_project_annotator(self.get_loggedin_request(), projects[0].pk, annotator.username) + add_project_annotator(self.get_loggedin_request(), projects[1].pk, annotator.username) + + # Make two more annotations + self.get_and_complete_annotation_task(annotator_request, projects[1]) + self.get_and_complete_annotation_task(annotator_request, projects[1]) + + # Remove annotator from 1st project, add to 2nd project + reject_project_annotator(self.get_loggedin_request(), projects[1].pk, annotator.username) + add_project_annotator(self.get_loggedin_request(), projects[2].pk, annotator.username) + + # Make two more annotations + self.get_and_complete_annotation_task(annotator_request, projects[2]) + self.get_and_complete_annotation_task(annotator_request, projects[2]) + + # Check no. of associated annotations + self.assertEqual(7, Annotation.objects.filter(user=annotator).count(), + "Annotator should have 7 associated annotations") + + + +