feat: test incorrect response file type

grader/grader.py

@@ -53,20 +53,32 @@ def validate_body(self):
         """Validate that the assignment's body is a dict with the correct keys."""
         body = self.assignment.get("body")
         if not isinstance(body, dict):
-            raise IncorrectResponseTypeError("The assignment's body must be a dict.")
+            body_type = type(body)
+            raise IncorrectResponseTypeError(f"The assignment's body must be a dict. Received {body_type}.")
         if not "chat_memory" in body:
-            raise InvalidResponseStructureError(f"The assignment's body must have a key named chat_memory. {body}")
+            raise InvalidResponseStructureError(
+                f"The assignment's body must have a key named chat_memory. body: {body}"
+            )
         if not "messages" in body["chat_memory"]:
-            raise InvalidResponseStructureError("The assignment's body.chat_memory must has a key named messages.")
+            raise InvalidResponseStructureError(
+                f"The assignment's body.chat_memory must has a key named messages. body: {body}"
+            )
         messages = body["chat_memory"]["messages"]
         if not isinstance(messages, list):
-            raise IncorrectResponseTypeError("The assignment's body.chat_memory.messages must be a list.")
+            messages_type = type(messages)
+            raise IncorrectResponseTypeError(
+                f"The assignment's body.chat_memory.messages must be a list. Received {messages_type}."
+            )
         if len(messages) < 2:
-            raise InvalidResponseStructureError("The messages list must contain at least two elements.")
+            raise InvalidResponseStructureError(
+                f"The messages list must contain at least two elements. messages: {messages}"
+            )
 
         for message in messages:
             if not isinstance(message, dict):
-                raise IncorrectResponseTypeError("All elements in the messages list must be dictionaries.")
+                raise IncorrectResponseTypeError(
+                    f"All elements in the messages list must be dictionaries. messages: {messages}"
+                )
 
         human_prompt = messages[0]
         ai_response = messages[1]
@@ -75,22 +87,22 @@ def validate_body(self):
         self.validate_keys(ai_response, AI_RESPONSE)
 
         if not human_prompt["type"] == "human":
-            raise IncorrectResponseValueError("The first message must be a human prompt.")
+            raise IncorrectResponseValueError(f"The first message must be a human prompt. first prompt: {human_prompt}")
         if not ai_response["type"] == "ai":
-            raise IncorrectResponseValueError("The second message must be an AI response.")
+            raise IncorrectResponseValueError(f"The second message must be an AI response. response: {ai_response}")
 
     def validate_metadata(self):
         """Validate that the assignment's metadata is a dict with the correct keys."""
         body = self.assignment.get("body")
         request_meta_data = body["request_meta_data"]
         if not isinstance(request_meta_data, dict):
-            raise InvalidResponseStructureError("The assignment must has a dict named request_meta_data.")
+            raise InvalidResponseStructureError(f"The assignment must has a dict named request_meta_data. body: {body}")
         if not request_meta_data["lambda"] == "lambda_langchain":
-            raise IncorrectResponseValueError("The request_meta_data.lambda must be lambda_langchain.")
+            raise IncorrectResponseValueError(f"The request_meta_data.lambda must be lambda_langchain. body: {body}")
         if not request_meta_data["model"] == "gpt-3.5-turbo":
-            raise IncorrectResponseValueError("The request_meta_data.model must be gpt-3.5-turbo.")
+            raise IncorrectResponseValueError(f"The request_meta_data.model must be gpt-3.5-turbo. body: {body}")
         if not request_meta_data["end_point"] == "ChatCompletion":
-            raise IncorrectResponseValueError("The request_meta_data.end_point must be ChatCompletion.")
+            raise IncorrectResponseValueError(f"The request_meta_data.end_point must be ChatCompletion. body: {body}")
 
     def validate(self):
         """Validate the assignment data structure."""

grader/tests/events/lawrence-mcdaniel-homework1-incorrect-response-type.txt

@@ -0,0 +1,35 @@
+"{
+  "isBase64Encoded": false,
+  "statusCode": 200,
+  "body": {
+    "chat_memory": {
+      "messages": [
+        {
+          "content": "Marv, I'd like to introduce you to all the nice YouTube viewers.",
+          "additional_kwargs": {},
+          "type": "human",
+          "example": false
+        },
+        {
+          "content": "Oh, how delightful. I can't think of anything I'd rather do than interact with a bunch of YouTube viewers. Just kidding, I'd rather be doing literally anything else. But go ahead, introduce me to your lovely audience. I'm sure they'll be absolutely thrilled to meet me.",
+          "additional_kwargs": {},
+          "type": "ai",
+          "example": false
+        }
+      ]
+    },
+    "output_key": null,
+    "input_key": null,
+    "return_messages": true,
+    "human_prefix": "Human",
+    "ai_prefix": "AI",
+    "memory_key": "chat_history",
+    "request_meta_data": {
+      "lambda": "lambda_langchain",
+      "model": "gpt-3.5-turbo",
+      "end_point": "ChatCompletion",
+      "temperature": 0.5,
+      "max_tokens": 256
+    }
+  }
+}"

grader/tests/init.py

@@ -6,5 +6,9 @@
 def get_event(filespec):
     """Reads a JSON file and returns the event"""
     with open(filespec, "r", encoding="utf-8") as f:  # pylint: disable=invalid-name
-        event = json.load(f)
-        return event
+        try:
+            event = json.load(f)
+            return event
+        except json.JSONDecodeError:
+            print(f"warning: invalid JSON in file: {filespec}")
+            return f.read()

grader/tests/test_responses.py

@@ -6,6 +6,9 @@
 """
 import pytest  # pylint: disable=unused-import
 
+from ..exceptions import (  # IncorrectResponseTypeError,; IncorrectResponseValueError,; ResponseFailedError,
+    InvalidResponseStructureError,
+)
 from ..grader import AutomatedGrader
 from .init import get_event
 
@@ -14,7 +17,7 @@
 class TestGrader:
     """Test the OpenAI API via Langchain using the Lambda Layer, 'genai'."""
 
-    def test_basic_request(self):
+    def test_success(self):
         """Test a valid successful submission."""
         assignment = get_event("tests/events/lawrence-mcdaniel-homework1-correct.json")
         automated_grader = AutomatedGrader(assignment=assignment)
@@ -28,3 +31,10 @@ def test_basic_request(self):
         assert "message" in grade, "The dictionary does not contain the key 'message'"
         assert isinstance(grade["message"], str), "The message is not a string"
         assert grade["message"] == "Great job!", "The message is not 'Great job!'"
+
+    def test_incorrect_response_type(self):
+        """Test an assignment with an incorrect response type."""
+        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-incorrect-response-type.txt")
+        automated_grader = AutomatedGrader(assignment=assignment)
+        with pytest.raises(InvalidResponseStructureError):
+            automated_grader.grade()