refactor: AutomatedGrader should always return the same kind of dict

grader/grader.py

@@ -39,7 +39,7 @@ def validate_statuscode(self):
         if not isinstance(self.assignment.get("statusCode"), int):
             raise IncorrectResponseTypeError("The assignment's statusCode must be an integer.")
         if not self.assignment["statusCode"] == 200:
-            raise IncorrectResponseValueError("The assignment's statusCode must be 200.")
+            raise ResponseFailedError("The assignment's statusCode must be 200.")
         return True
 
     def validate_base64encoded(self):
@@ -114,24 +114,26 @@ def validate(self):
         self.validate_body()
         self.validate_metadata()
 
-    def grade_response(self, grade, message):
+    def grade_response(self, grade, message=None):
         """Create a grade dict from the assignment."""
+        message_type = message.__class__.__name__ if message else "Success"
+        message = str(message) if message else "Great job!"
         return {
             "grade": grade,
             "message": message,
+            "message_type": message_type,
         }
 
     def grade(self):
         """Grade the assignment."""
-        self.validate()
         try:
             self.validate()
         except InvalidResponseStructureError as e:
-            return self.grade_response(75, e.message)
+            return self.grade_response(75, e)
         except ResponseFailedError as e:
-            return self.grade_response(80, e.message)
+            return self.grade_response(80, e)
         except IncorrectResponseValueError as e:
-            return self.grade_response(85, e.message)
+            return self.grade_response(85, e)
         except IncorrectResponseTypeError as e:
-            return self.grade_response(90, e.message)
-        return self.grade_response(100, "Great job!")
+            return self.grade_response(90, e)
+        return self.grade_response(100)

grader/tests/events/lawrence-mcdaniel-homework1-correct-verbose.json

@@ -0,0 +1,59 @@
+{
+  "isBase64Encoded": false,
+  "statusCode": 200,
+  "body": {
+    "chat_memory": {
+      "messages": [
+        {
+          "content": "Marv, I'd like to introduce you to all the nice YouTube viewers.",
+          "additional_kwargs": {},
+          "type": "human",
+          "example": false
+        },
+        {
+          "content": "Oh, how delightful. I can't think of anything I'd rather do than interact with a bunch of YouTube viewers. Just kidding, I'd rather be doing literally anything else. But go ahead, introduce me to your lovely audience. I'm sure they'll be absolutely thrilled to meet me.",
+          "additional_kwargs": {},
+          "type": "ai",
+          "example": false
+        },
+        {
+          "content": "Marv, I'd like to introduce you to all the nice YouTube viewers.",
+          "additional_kwargs": {},
+          "type": "human",
+          "example": false
+        },
+        {
+          "content": "Oh, how delightful. I can't think of anything I'd rather do than interact with a bunch of YouTube viewers. Just kidding, I'd rather be doing literally anything else. But go ahead, introduce me to your lovely audience. I'm sure they'll be absolutely thrilled to meet me.",
+          "additional_kwargs": {},
+          "type": "ai",
+          "example": false
+        },
+        {
+          "content": "Marv, I'd like to introduce you to all the nice YouTube viewers.",
+          "additional_kwargs": {},
+          "type": "human",
+          "example": false
+        },
+        {
+          "content": "Oh, how delightful. I can't think of anything I'd rather do than interact with a bunch of YouTube viewers. Just kidding, I'd rather be doing literally anything else. But go ahead, introduce me to your lovely audience. I'm sure they'll be absolutely thrilled to meet me.",
+          "additional_kwargs": {},
+          "type": "ai",
+          "example": false
+        }
+      ]
+    },
+    "output_key": null,
+    "input_key": null,
+    "return_messages": true,
+    "human_prefix": "Human",
+    "ai_prefix": "AI",
+    "memory_key": "chat_history",
+    "request_meta_data": {
+      "lambda": "lambda_langchain",
+      "model": "gpt-3.5-turbo",
+      "end_point": "ChatCompletion",
+      "temperature": 0.5,
+      "max_tokens": 256
+    }
+  }
+}

grader/tests/events/lawrence-mcdaniel-homework1-incorrect-response-status.json

@@ -1,6 +1,6 @@
 {
   "isBase64Encoded": false,
-  "statusCode": 200,
+  "statusCode": 403,
   "body": {
     "chat_memory": {
       "messages": [

grader/tests/events/lawrence-mcdaniel-homework1-invalid-json.json → grader/tests/events/lawrence-mcdaniel-homework1-type-error.json

@@ -1,6 +1,6 @@
 {
   "isBase64Encoded": false,
-  "statusCode": 200,
+  "statusCode": "200",
   "body": {
     "chat_memory": {
       "messages": [

grader/tests/events/lawrence-mcdaniel-homework1-wrong-messages.json

@@ -0,0 +1,35 @@
+{
+  "isBase64Encoded": false,
+  "statusCode": 200,
+  "body": {
+    "chat_memory": {
+      "messages": [
+        {
+          "content": "no idea what to say",
+          "additional_kwargs": {},
+          "type": "wilderbeast",
+          "example": false
+        },
+        {
+          "content": "me neither",
+          "additional_kwargs": {},
+          "type": "assistant",
+          "example": false
+        }
+      ]
+    },
+    "output_key": null,
+    "input_key": null,
+    "return_messages": true,
+    "human_prefix": "Human",
+    "ai_prefix": "AI",
+    "memory_key": "chat_history",
+    "request_meta_data": {
+      "lambda": "lambda_langchain",
+      "model": "gpt-3.5-turbo",
+      "end_point": "ChatCompletion",
+      "temperature": 0.5,
+      "max_tokens": 256
+    }
+  }
+}

grader/tests/test_responses.py

@@ -6,9 +6,6 @@
 """
 import pytest  # pylint: disable=unused-import
 
-from ..exceptions import (  # IncorrectResponseTypeError,; IncorrectResponseValueError,; ResponseFailedError,
-    InvalidResponseStructureError,
-)
 from ..grader import AutomatedGrader
 from .init import get_event
 
@@ -32,9 +29,54 @@ def test_success(self):
         assert isinstance(grade["message"], str), "The message is not a string"
         assert grade["message"] == "Great job!", "The message is not 'Great job!'"
 
+    def test_success_verbose(self):
+        """Test a valid successful submission."""
+        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-correct-verbose.json")
+        automated_grader = AutomatedGrader(assignment=assignment)
+        grade = automated_grader.grade()
+
+        assert isinstance(grade, dict), "The grade is not a dictionary"
+        assert "grade" in grade, "The dictionary does not contain the key 'grade'"
+        assert isinstance(grade["grade"], int), "The grade is not an int"
+        assert grade["grade"] == 100, "The grade is not 100"
+
+        assert "message" in grade, "The dictionary does not contain the key 'message'"
+        assert isinstance(grade["message"], str), "The message is not a string"
+        assert grade["message"] == "Great job!", "The message is not 'Great job!'"
+
     def test_incorrect_response_type(self):
         """Test an assignment with an incorrect response type."""
         assignment = get_event("tests/events/lawrence-mcdaniel-homework1-incorrect-response-type.txt")
         automated_grader = AutomatedGrader(assignment=assignment)
-        with pytest.raises(InvalidResponseStructureError):
-            automated_grader.grade()
+        grade = automated_grader.grade()
+        print(grade)
+
+        assert grade["grade"] == 75, "The grade is not 75"
+        assert grade["message_type"] == "InvalidResponseStructureError"
+
+    def test_incorrect_response_statuscode(self):
+        """Test an assignment with an incorrect response status code."""
+        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-incorrect-response-status.json")
+        automated_grader = AutomatedGrader(assignment=assignment)
+
+        grade = automated_grader.grade()
+        assert grade["message_type"] == "ResponseFailedError"
+        assert grade["grade"] == 80, "The grade is not 80"
+
+    def test_incorrect_messages(self):
+        """Test an assignment with an incorrect message."""
+        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-wrong-messages.json")
+        automated_grader = AutomatedGrader(assignment=assignment)
+
+        grade = automated_grader.grade()
+        assert grade["message_type"] == "IncorrectResponseValueError"
+        assert grade["grade"] == 85, "The grade is not 85"
+
+    def test_incorrect_data_type(self):
+        """Test an assignment with an incorrect data type."""
+        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-type-error.json")
+        automated_grader = AutomatedGrader(assignment=assignment)
+
+        grade = automated_grader.grade()
+        assert grade["message_type"] == "IncorrectResponseTypeError"
+        assert grade["grade"] == 90, "The grade is not 85"