From e0477ddebf4b6659f2b7282b79d089879ef6a5fe Mon Sep 17 00:00:00 2001 From: lpm0073 Date: Mon, 27 Nov 2023 17:40:11 -0600 Subject: [PATCH] refactor: AutomatedGrader should always return the same kind of dict --- grader/grader.py | 18 +++--- ...ce-mcdaniel-homework1-correct-verbose.json | 59 +++++++++++++++++++ ...l-homework1-incorrect-response-status.json | 2 +- ...wrence-mcdaniel-homework1-type-error.json} | 2 +- ...nce-mcdaniel-homework1-wrong-messages.json | 35 +++++++++++ grader/tests/test_responses.py | 52 ++++++++++++++-- 6 files changed, 153 insertions(+), 15 deletions(-) create mode 100644 grader/tests/events/lawrence-mcdaniel-homework1-correct-verbose.json rename grader/tests/events/{lawrence-mcdaniel-homework1-invalid-json.json => lawrence-mcdaniel-homework1-type-error.json} (97%) create mode 100644 grader/tests/events/lawrence-mcdaniel-homework1-wrong-messages.json diff --git a/grader/grader.py b/grader/grader.py index ae2d0a5..38b2a9a 100644 --- a/grader/grader.py +++ b/grader/grader.py @@ -39,7 +39,7 @@ def validate_statuscode(self): if not isinstance(self.assignment.get("statusCode"), int): raise IncorrectResponseTypeError("The assignment's statusCode must be an integer.") if not self.assignment["statusCode"] == 200: - raise IncorrectResponseValueError("The assignment's statusCode must be 200.") + raise ResponseFailedError("The assignment's statusCode must be 200.") return True def validate_base64encoded(self): @@ -114,24 +114,26 @@ def validate(self): self.validate_body() self.validate_metadata() - def grade_response(self, grade, message): + def grade_response(self, grade, message=None): """Create a grade dict from the assignment.""" + message_type = message.__class__.__name__ if message else "Success" + message = str(message) if message else "Great job!" return { "grade": grade, "message": message, + "message_type": message_type, } def grade(self): """Grade the assignment.""" - self.validate() try: self.validate() except InvalidResponseStructureError as e: - return self.grade_response(75, e.message) + return self.grade_response(75, e) except ResponseFailedError as e: - return self.grade_response(80, e.message) + return self.grade_response(80, e) except IncorrectResponseValueError as e: - return self.grade_response(85, e.message) + return self.grade_response(85, e) except IncorrectResponseTypeError as e: - return self.grade_response(90, e.message) - return self.grade_response(100, "Great job!") + return self.grade_response(90, e) + return self.grade_response(100) diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-correct-verbose.json b/grader/tests/events/lawrence-mcdaniel-homework1-correct-verbose.json new file mode 100644 index 0000000..db82864 --- /dev/null +++ b/grader/tests/events/lawrence-mcdaniel-homework1-correct-verbose.json @@ -0,0 +1,59 @@ +{ + "isBase64Encoded": false, + "statusCode": 200, + "body": { + "chat_memory": { + "messages": [ + { + "content": "Marv, I'd like to introduce you to all the nice YouTube viewers.", + "additional_kwargs": {}, + "type": "human", + "example": false + }, + { + "content": "Oh, how delightful. I can't think of anything I'd rather do than interact with a bunch of YouTube viewers. Just kidding, I'd rather be doing literally anything else. But go ahead, introduce me to your lovely audience. I'm sure they'll be absolutely thrilled to meet me.", + "additional_kwargs": {}, + "type": "ai", + "example": false + }, + { + "content": "Marv, I'd like to introduce you to all the nice YouTube viewers.", + "additional_kwargs": {}, + "type": "human", + "example": false + }, + { + "content": "Oh, how delightful. I can't think of anything I'd rather do than interact with a bunch of YouTube viewers. Just kidding, I'd rather be doing literally anything else. But go ahead, introduce me to your lovely audience. I'm sure they'll be absolutely thrilled to meet me.", + "additional_kwargs": {}, + "type": "ai", + "example": false + }, + { + "content": "Marv, I'd like to introduce you to all the nice YouTube viewers.", + "additional_kwargs": {}, + "type": "human", + "example": false + }, + { + "content": "Oh, how delightful. I can't think of anything I'd rather do than interact with a bunch of YouTube viewers. Just kidding, I'd rather be doing literally anything else. But go ahead, introduce me to your lovely audience. I'm sure they'll be absolutely thrilled to meet me.", + "additional_kwargs": {}, + "type": "ai", + "example": false + } + ] + }, + "output_key": null, + "input_key": null, + "return_messages": true, + "human_prefix": "Human", + "ai_prefix": "AI", + "memory_key": "chat_history", + "request_meta_data": { + "lambda": "lambda_langchain", + "model": "gpt-3.5-turbo", + "end_point": "ChatCompletion", + "temperature": 0.5, + "max_tokens": 256 + } + } +} diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-incorrect-response-status.json b/grader/tests/events/lawrence-mcdaniel-homework1-incorrect-response-status.json index 9be6b48..ed101cb 100644 --- a/grader/tests/events/lawrence-mcdaniel-homework1-incorrect-response-status.json +++ b/grader/tests/events/lawrence-mcdaniel-homework1-incorrect-response-status.json @@ -1,6 +1,6 @@ { "isBase64Encoded": false, - "statusCode": 200, + "statusCode": 403, "body": { "chat_memory": { "messages": [ diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-invalid-json.json b/grader/tests/events/lawrence-mcdaniel-homework1-type-error.json similarity index 97% rename from grader/tests/events/lawrence-mcdaniel-homework1-invalid-json.json rename to grader/tests/events/lawrence-mcdaniel-homework1-type-error.json index 9be6b48..bff4462 100644 --- a/grader/tests/events/lawrence-mcdaniel-homework1-invalid-json.json +++ b/grader/tests/events/lawrence-mcdaniel-homework1-type-error.json @@ -1,6 +1,6 @@ { "isBase64Encoded": false, - "statusCode": 200, + "statusCode": "200", "body": { "chat_memory": { "messages": [ diff --git a/grader/tests/events/lawrence-mcdaniel-homework1-wrong-messages.json b/grader/tests/events/lawrence-mcdaniel-homework1-wrong-messages.json new file mode 100644 index 0000000..efdf624 --- /dev/null +++ b/grader/tests/events/lawrence-mcdaniel-homework1-wrong-messages.json @@ -0,0 +1,35 @@ +{ + "isBase64Encoded": false, + "statusCode": 200, + "body": { + "chat_memory": { + "messages": [ + { + "content": "no idea what to say", + "additional_kwargs": {}, + "type": "wilderbeast", + "example": false + }, + { + "content": "me neither", + "additional_kwargs": {}, + "type": "assistant", + "example": false + } + ] + }, + "output_key": null, + "input_key": null, + "return_messages": true, + "human_prefix": "Human", + "ai_prefix": "AI", + "memory_key": "chat_history", + "request_meta_data": { + "lambda": "lambda_langchain", + "model": "gpt-3.5-turbo", + "end_point": "ChatCompletion", + "temperature": 0.5, + "max_tokens": 256 + } + } +} diff --git a/grader/tests/test_responses.py b/grader/tests/test_responses.py index cbe3b53..59b47ea 100644 --- a/grader/tests/test_responses.py +++ b/grader/tests/test_responses.py @@ -6,9 +6,6 @@ """ import pytest # pylint: disable=unused-import -from ..exceptions import ( # IncorrectResponseTypeError,; IncorrectResponseValueError,; ResponseFailedError, - InvalidResponseStructureError, -) from ..grader import AutomatedGrader from .init import get_event @@ -32,9 +29,54 @@ def test_success(self): assert isinstance(grade["message"], str), "The message is not a string" assert grade["message"] == "Great job!", "The message is not 'Great job!'" + def test_success_verbose(self): + """Test a valid successful submission.""" + assignment = get_event("tests/events/lawrence-mcdaniel-homework1-correct-verbose.json") + automated_grader = AutomatedGrader(assignment=assignment) + grade = automated_grader.grade() + + assert isinstance(grade, dict), "The grade is not a dictionary" + assert "grade" in grade, "The dictionary does not contain the key 'grade'" + assert isinstance(grade["grade"], int), "The grade is not an int" + assert grade["grade"] == 100, "The grade is not 100" + + assert "message" in grade, "The dictionary does not contain the key 'message'" + assert isinstance(grade["message"], str), "The message is not a string" + assert grade["message"] == "Great job!", "The message is not 'Great job!'" + def test_incorrect_response_type(self): """Test an assignment with an incorrect response type.""" assignment = get_event("tests/events/lawrence-mcdaniel-homework1-incorrect-response-type.txt") automated_grader = AutomatedGrader(assignment=assignment) - with pytest.raises(InvalidResponseStructureError): - automated_grader.grade() + grade = automated_grader.grade() + print(grade) + + assert grade["grade"] == 75, "The grade is not 75" + assert grade["message_type"] == "InvalidResponseStructureError" + + def test_incorrect_response_statuscode(self): + """Test an assignment with an incorrect response status code.""" + assignment = get_event("tests/events/lawrence-mcdaniel-homework1-incorrect-response-status.json") + automated_grader = AutomatedGrader(assignment=assignment) + + grade = automated_grader.grade() + assert grade["message_type"] == "ResponseFailedError" + assert grade["grade"] == 80, "The grade is not 80" + + def test_incorrect_messages(self): + """Test an assignment with an incorrect message.""" + assignment = get_event("tests/events/lawrence-mcdaniel-homework1-wrong-messages.json") + automated_grader = AutomatedGrader(assignment=assignment) + + grade = automated_grader.grade() + assert grade["message_type"] == "IncorrectResponseValueError" + assert grade["grade"] == 85, "The grade is not 85" + + def test_incorrect_data_type(self): + """Test an assignment with an incorrect data type.""" + assignment = get_event("tests/events/lawrence-mcdaniel-homework1-type-error.json") + automated_grader = AutomatedGrader(assignment=assignment) + + grade = automated_grader.grade() + assert grade["message_type"] == "IncorrectResponseTypeError" + assert grade["grade"] == 90, "The grade is not 85"