Merge pull request #8 from lpm0073/next

add more unit tests

.github/actions/tests/pre-commit/action.yml

@@ -71,7 +71,8 @@ runs:
         pip install -r ./requirements.txt
 
     # see: https://pre-commit.ci/lite.html
-    - name: pre-commit ci
-      id: pre-commit-ci
-      if: always()
-      uses: pre-commit-ci/[email protected]
+    - name: pre-commit
+      id: pre-commit
+      shell: bash
+      run: |
+        pre-commit run --all-files --show-diff-on-failure

Makefile

@@ -25,15 +25,15 @@ activate:
 	. venv/bin/activate
 
 test:
-	cd grader && \
-	pytest -v -s tests/
+	cd grader && pytest -v -s tests/
+	python -m setup_test
 
 lint:
 	pre-commit run --all-files && \
 	black .
 
 clean:
-	rm -rf venv
+	rm -rf venv && rm -rf node_modules
 
 
 ######################

grader/grader.py

@@ -22,7 +22,7 @@ class AutomatedGrader:
 
     def __init__(self, assignment):
         self.assignment = assignment
-        with open(REQUIRED_KEYS_SPEC, "r", encoding="utf-8") as f:  # pylint: disable=invalid-name
+        with open("data/" + REQUIRED_KEYS_SPEC, "r", encoding="utf-8") as f:  # pylint: disable=invalid-name
             self.required_keys = json.load(f)
 
     def validate_keys(self, subject, control):
@@ -31,26 +31,46 @@ def validate_keys(self, subject, control):
         required_keys = set(control.keys())
 
         if not required_keys.issubset(assignment_keys):
-            raise InvalidResponseStructureError("The assignment is missing one or more required keys.")
+            missing_keys = required_keys.difference(assignment_keys)
+            raise InvalidResponseStructureError(
+                f"The assignment is missing one or more required keys. missing: {missing_keys}"
+            )
         return True
 
     def validate_statuscode(self):
         """Validate that the assignment's statusCode is 200."""
+        if "statusCode" not in self.assignment:
+            raise InvalidResponseStructureError(f"The assignment must have a statusCode. assignment: {self.assignment}")
         if not isinstance(self.assignment.get("statusCode"), int):
-            raise IncorrectResponseTypeError("The assignment's statusCode must be an integer.")
-        if not self.assignment["statusCode"] == 200:
-            raise ResponseFailedError("The assignment's statusCode must be 200.")
+            status_code_type = type(self.assignment.get("statusCode"))
+            raise IncorrectResponseTypeError(
+                f"The assignment's statusCode must be an integer. received: {status_code_type}"
+            )
+        status_code = self.assignment["statusCode"]
+        if not status_code == 200:
+            raise ResponseFailedError(f"The assignment's statusCode must be 200. received: {status_code}")
         return True
 
     def validate_base64encoded(self):
         """Validate that the assignment's isBase64Encoded is False."""
-        if not isinstance(self.assignment.get("isBase64Encoded"), bool):
-            raise IncorrectResponseTypeError("The assignment's base64Encoded must be a boolean.")
+        if "isBase64Encoded" not in self.assignment:
+            raise InvalidResponseStructureError(
+                f"The assignment must have a isBase64Encoded. assignment: {self.assignment}"
+            )
+        is_base64_encoded = self.assignment.get("isBase64Encoded")
+        if not isinstance(is_base64_encoded, bool):
+            is_base64_encoded_type = type(is_base64_encoded)
+            raise IncorrectResponseTypeError(
+                f"The assignment's base64Encoded must be a boolean. received: {is_base64_encoded_type}"
+            )
         if self.assignment["isBase64Encoded"]:
             raise IncorrectResponseValueError("The assignment's isBase64Encoded must be False.")
 
     def validate_body(self):
         """Validate that the assignment's body is a dict with the correct keys."""
+        if "body" not in self.assignment:
+            raise InvalidResponseStructureError(f"The assignment must have a body. assignment: {self.assignment}")
+
         body = self.assignment.get("body")
         if not isinstance(body, dict):
             body_type = type(body)
@@ -76,7 +96,7 @@ def validate_body(self):
 
         for message in messages:
             if not isinstance(message, dict):
-                raise IncorrectResponseTypeError(
+                raise InvalidResponseStructureError(
                     f"All elements in the messages list must be dictionaries. messages: {messages}"
                 )
 
@@ -96,12 +116,28 @@ def validate_metadata(self):
         body = self.assignment.get("body")
         request_meta_data = body["request_meta_data"]
         if not isinstance(request_meta_data, dict):
-            raise InvalidResponseStructureError(f"The assignment must has a dict named request_meta_data. body: {body}")
-        if not request_meta_data["lambda"] == "lambda_langchain":
+            meta_data_type = type(request_meta_data)
+            raise InvalidResponseStructureError(
+                f"The assignment must has a dict named request_meta_data. received: {meta_data_type}"
+            )
+        if request_meta_data.get("lambda") is None:
+            raise InvalidResponseStructureError(
+                f"The request_meta_data key lambda_langchain must exist. request_meta_data: {request_meta_data}"
+            )
+        if request_meta_data.get("model") is None:
+            raise InvalidResponseStructureError(
+                f"The request_meta_data key model must exist. request_meta_data: {request_meta_data}"
+            )
+        if request_meta_data.get("end_point") is None:
+            raise InvalidResponseStructureError(
+                f"The request_meta_data end_point must exist. request_meta_data: {request_meta_data}"
+            )
+
+        if not request_meta_data.get("lambda") == "lambda_langchain":
             raise IncorrectResponseValueError(f"The request_meta_data.lambda must be lambda_langchain. body: {body}")
-        if not request_meta_data["model"] == "gpt-3.5-turbo":
+        if not request_meta_data.get("model") == "gpt-3.5-turbo":
             raise IncorrectResponseValueError(f"The request_meta_data.model must be gpt-3.5-turbo. body: {body}")
-        if not request_meta_data["end_point"] == "ChatCompletion":
+        if not request_meta_data.get("end_point") == "ChatCompletion":
             raise IncorrectResponseValueError(f"The request_meta_data.end_point must be ChatCompletion. body: {body}")
 
     def validate(self):
@@ -129,7 +165,7 @@ def grade(self):
         try:
             self.validate()
         except InvalidResponseStructureError as e:
-            return self.grade_response(75, e)
+            return self.grade_response(70, e)
         except ResponseFailedError as e:
             return self.grade_response(80, e)
         except IncorrectResponseValueError as e:

grader/tests/events/bad-data.txt

@@ -0,0 +1 @@
+completely totally messed up bad data

grader/tests/events/lawrence-mcdaniel-homework1-bad-message-1.json

@@ -0,0 +1,32 @@
+{
+  "isBase64Encoded": false,
+  "statusCode": 200,
+  "body": {
+    "chat_memory": {
+      "messages": [
+        {
+          "content": "Marv, I'd like to introduce you to all the nice YouTube viewers."
+        },
+        {
+          "content": "Oh, how delightful. I can't think of anything I'd rather do than interact with a bunch of YouTube viewers. Just kidding, I'd rather be doing literally anything else. But go ahead, introduce me to your lovely audience. I'm sure they'll be absolutely thrilled to meet me.",
+          "additional_kwargs": {},
+          "type": "ai",
+          "example": false
+        }
+      ]
+    },
+    "output_key": null,
+    "input_key": null,
+    "return_messages": true,
+    "human_prefix": "Human",
+    "ai_prefix": "AI",
+    "memory_key": "chat_history",
+    "request_meta_data": {
+      "lambda": "lambda_langchain",
+      "model": "gpt-3.5-turbo",
+      "end_point": "ChatCompletion",
+      "temperature": 0.5,
+      "max_tokens": 256
+    }
+  }
+}

grader/tests/events/lawrence-mcdaniel-homework1-bad-message-2.json

@@ -0,0 +1,29 @@
+{
+  "isBase64Encoded": false,
+  "statusCode": 200,
+  "body": {
+    "chat_memory": {
+      "messages": [
+        {
+          "content": "Oh, how delightful. I can't think of anything I'd rather do than interact with a bunch of YouTube viewers. Just kidding, I'd rather be doing literally anything else. But go ahead, introduce me to your lovely audience. I'm sure they'll be absolutely thrilled to meet me.",
+          "additional_kwargs": {},
+          "type": "ai",
+          "example": false
+        }
+      ]
+    },
+    "output_key": null,
+    "input_key": null,
+    "return_messages": true,
+    "human_prefix": "Human",
+    "ai_prefix": "AI",
+    "memory_key": "chat_history",
+    "request_meta_data": {
+      "lambda": "lambda_langchain",
+      "model": "gpt-3.5-turbo",
+      "end_point": "ChatCompletion",
+      "temperature": 0.5,
+      "max_tokens": 256
+    }
+  }
+}

grader/tests/events/lawrence-mcdaniel-homework1-bad-message-3.json

@@ -0,0 +1,22 @@
+{
+  "isBase64Encoded": false,
+  "statusCode": 200,
+  "body": {
+    "chat_memory": {
+      "messages": ["bad", "data"]
+    },
+    "output_key": null,
+    "input_key": null,
+    "return_messages": true,
+    "human_prefix": "Human",
+    "ai_prefix": "AI",
+    "memory_key": "chat_history",
+    "request_meta_data": {
+      "lambda": "lambda_langchain",
+      "model": "gpt-3.5-turbo",
+      "end_point": "ChatCompletion",
+      "temperature": 0.5,
+      "max_tokens": 256
+    }
+  }
+}

grader/tests/events/lawrence-mcdaniel-homework1-bad-message-4.json

@@ -0,0 +1,29 @@
+{
+  "isBase64Encoded": false,
+  "statusCode": 200,
+  "body": {
+    "chat_memory": {
+      "messages": [
+        {
+          "content": "Marv, I'd like to introduce you to all the nice YouTube viewers.",
+          "additional_kwargs": {},
+          "type": "human",
+          "example": false
+        },
+        {
+          "content": "Oh, how delightful. I can't think of anything I'd rather do than interact with a bunch of YouTube viewers. Just kidding, I'd rather be doing literally anything else. But go ahead, introduce me to your lovely audience. I'm sure they'll be absolutely thrilled to meet me.",
+          "additional_kwargs": {},
+          "type": "ai",
+          "example": false
+        }
+      ]
+    },
+    "output_key": null,
+    "input_key": null,
+    "return_messages": true,
+    "human_prefix": "Human",
+    "ai_prefix": "AI",
+    "memory_key": "chat_history",
+    "request_meta_data": {}
+  }
+}

grader/tests/test_responses.py

@@ -21,6 +21,7 @@ def test_success(self):
         grade = automated_grader.grade()
 
         assert isinstance(grade, dict), "The grade is not a dictionary"
+        assert grade["message_type"] == "Success"
         assert "grade" in grade, "The dictionary does not contain the key 'grade'"
         assert isinstance(grade["grade"], int), "The grade is not an int"
         assert grade["grade"] == 100, "The grade is not 100"
@@ -36,6 +37,7 @@ def test_success_verbose(self):
         grade = automated_grader.grade()
 
         assert isinstance(grade, dict), "The grade is not a dictionary"
+        assert grade["message_type"] == "Success"
         assert "grade" in grade, "The dictionary does not contain the key 'grade'"
         assert isinstance(grade["grade"], int), "The grade is not an int"
         assert grade["grade"] == 100, "The grade is not 100"
@@ -44,14 +46,24 @@ def test_success_verbose(self):
         assert isinstance(grade["message"], str), "The message is not a string"
         assert grade["message"] == "Great job!", "The message is not 'Great job!'"
 
+    def test_bad_data(self):
+        """Test an assignment with bad data."""
+        assignment = get_event("tests/events/bad-data.txt")
+        automated_grader = AutomatedGrader(assignment=assignment)
+        grade = automated_grader.grade()
+        print(grade)
+
+        assert grade["grade"] == 70, "The grade is not 70"
+        assert grade["message_type"] == "InvalidResponseStructureError"
+
     def test_incorrect_response_type(self):
         """Test an assignment with an incorrect response type."""
         assignment = get_event("tests/events/lawrence-mcdaniel-homework1-incorrect-response-type.txt")
         automated_grader = AutomatedGrader(assignment=assignment)
         grade = automated_grader.grade()
         print(grade)
 
-        assert grade["grade"] == 75, "The grade is not 75"
+        assert grade["grade"] == 70, "The grade is not 70"
         assert grade["message_type"] == "InvalidResponseStructureError"
 
     def test_incorrect_response_statuscode(self):
@@ -80,3 +92,39 @@ def test_incorrect_data_type(self):
         grade = automated_grader.grade()
         assert grade["message_type"] == "IncorrectResponseTypeError"
         assert grade["grade"] == 90, "The grade is not 85"
+
+    def test_bad_message_01(self):
+        """Test an assignment with an incorrect message."""
+        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-bad-message-1.json")
+        automated_grader = AutomatedGrader(assignment=assignment)
+
+        grade = automated_grader.grade()
+        assert grade["message_type"] == "InvalidResponseStructureError"
+        assert grade["grade"] == 70, "The grade is not 70"
+
+    def test_bad_message_02(self):
+        """Test an assignment with an incorrect message."""
+        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-bad-message-2.json")
+        automated_grader = AutomatedGrader(assignment=assignment)
+
+        grade = automated_grader.grade()
+        assert grade["message_type"] == "InvalidResponseStructureError"
+        assert grade["grade"] == 70, "The grade is not 70"
+
+    def test_bad_message_03(self):
+        """Test an assignment with an incorrect message."""
+        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-bad-message-3.json")
+        automated_grader = AutomatedGrader(assignment=assignment)
+
+        grade = automated_grader.grade()
+        assert grade["message_type"] == "InvalidResponseStructureError"
+        assert grade["grade"] == 70, "The grade is not 70"
+
+    def test_bad_message_04(self):
+        """Test an assignment with an incorrect message."""
+        assignment = get_event("tests/events/lawrence-mcdaniel-homework1-bad-message-4.json")
+        automated_grader = AutomatedGrader(assignment=assignment)
+
+        grade = automated_grader.grade()
+        assert grade["message_type"] == "InvalidResponseStructureError"
+        assert grade["grade"] == 70, "The grade is not 70"

grader/setup.py → setup.py

@@ -1,8 +1,9 @@
 # -*- coding: utf-8 -*-
 """Setup for automated_grader package."""
-from setup_utils import get_semantic_version  # pylint: disable=import-error
 from setuptools import find_packages, setup
 
+from setup_utils import get_semantic_version  # pylint: disable=import-error
+
 
 setup(
     name="automated_grader",
@@ -12,7 +13,7 @@
     author_email="[email protected]",
     packages=find_packages(),
     package_data={
-        "automated_grader": ["*.md", "data/*"],
+        "automated_grader": ["*.md"],
     },
     install_requires=[],
 )

grader/setup_utils.py → setup_utils.py

@@ -9,14 +9,15 @@
 
 
 HERE = os.path.abspath(os.path.dirname(__file__))
+PROJECT_ROOT = os.path.abspath(os.path.join(HERE, "grader"))
 
 # allow setup.py to be run from any path
 os.chdir(os.path.normpath(os.path.join(os.path.abspath(__file__), os.pardir)))
 
 
 def load_version() -> Dict[str, str]:
     """Stringify the __version__ module."""
-    version_file_path = os.path.join(HERE, "__version__.py")
+    version_file_path = os.path.join(PROJECT_ROOT, "__version__.py")
     spec = importlib.util.spec_from_file_location("__version__", version_file_path)
     version_module = importlib.util.module_from_spec(spec)
     spec.loader.exec_module(version_module)