Merge branch 'main' into next-major

README.md

@@ -14,9 +14,24 @@ make activate
 ## Usage
 
 ```console
+# command-line help
+python3 -m grader.batch -h
+
+# example usage
 python3 -m grader.batch 'path/to/homework/json/files/'
 ```
 
+## Rubric
+
+Rubric values are expressed as floats between 0 and 1.00, and can be overridden with environment variables.
+
+```console
+AG_INCORRECT_RESPONSE_TYPE_PENALTY_PCT=0.10
+AG_INCORRECT_RESPONSE_VALUE_PENALTY_PCT=0.15
+AG_RESPONSE_FAILED_PENALTY_PCT=0.20
+AG_INVALID_RESPONSE_STRUCTURE_PENALTY_PCT=0.30
+```
+
 ### Expected output
 
 ```console

grader/__version__.py

@@ -1,2 +1,2 @@
 # -*- coding: utf-8 -*-
-__version__ = "1.2.0"
+__version__ = "1.3.0"

grader/batch.py

@@ -8,7 +8,7 @@
 from .grader import AutomatedGrader
 
 
-def main(filepath: str = None, output_folder: str = "out"):
+def main(filepath: str = None, output_folder: str = "out", potential_points: int = 100):
     """Grade an assignment."""
     graded = 0
     if filepath is None:
@@ -26,7 +26,7 @@ def main(filepath: str = None, output_folder: str = "out"):
             except json.JSONDecodeError:
                 print(f"warning: invalid JSON in assignment_filename: {assignment_filename}")
                 assignment = f.read()
-        grader = AutomatedGrader(assignment)
+        grader = AutomatedGrader(assignment, potential_points=potential_points)
         grade = grader.grade()
         with open(
             os.path.join(OUTPUT_FILE_PATH, f"{os.path.basename(assignment_filename)}"), "w", encoding="utf-8"
@@ -47,7 +47,14 @@ def main(filepath: str = None, output_folder: str = "out"):
         default="out",
         help="The name of the subfolder where graded assignments will be saved.",
     )
+    parser.add_argument(
+        "potential_points",
+        type=int,
+        nargs="?",  # optional
+        default=100,
+        help="The aggregate point potential for the assignment.",
+    )
 
     args = parser.parse_args()
 
-    main(args.filepath, args.output_folder)
+    main(args.filepath, args.output_folder, args.potential_points)

grader/config.py

@@ -0,0 +1,22 @@
+# -*- coding: utf-8 -*-
+"""Setup for automated_grader package."""
+
+import os
+
+from dotenv import find_dotenv, load_dotenv
+
+
+# for local development and unit testing
+dotenv_path = find_dotenv()
+if os.path.exists(dotenv_path):
+    load_dotenv(dotenv_path=dotenv_path, verbose=True)
+
+
+# pylint: disable=too-few-public-methods
+class AGRubric:
+    """Constants for the AutomatedGrader class."""
+
+    INCORRECT_RESPONSE_TYPE_PENALTY_PCT = float(os.getenv("AG_INCORRECT_RESPONSE_TYPE_PENALTY_PCT", "0.10"))
+    INCORRECT_RESPONSE_VALUE_PENALTY_PCT = float(os.getenv("AG_INCORRECT_RESPONSE_VALUE_PENALTY_PCT", "0.15"))
+    RESPONSE_FAILED_PENALTY_PCT = float(os.getenv("AG_RESPONSE_FAILED_PENALTY_PCT", "0.20"))
+    INVALID_RESPONSE_STRUCTURE_PENALTY_PCT = float(os.getenv("AG_INVALID_RESPONSE_STRUCTURE_PENALTY_PCT", "0.30"))

grader/exceptions.py

@@ -1,34 +1,56 @@
 # -*- coding: utf-8 -*-
 """Custom exceptions for the AutomatedGrader class."""
 
+from .config import AGRubric
 
-class InvalidResponseStructureError(Exception):
+
+class AGException(Exception):
+    """A custom base exception for the AutomatedGrader class."""
+
+    def __init__(self, message: str, penalty_pct: float):
+        self.message = message
+
+        if not isinstance(penalty_pct, float):
+            raise TypeError(f"penalty_pct must be a float. penalty_pct: {penalty_pct}")
+        if not 0 <= penalty_pct <= 1:
+            raise ValueError(f"penalty_pct must be between 0 and 1. penalty_pct: {penalty_pct}")
+
+        self._penalty_pct = penalty_pct
+        super().__init__(self.message)
+
+    @property
+    def penalty_pct(self):
+        """Return the penalty percentage for this error."""
+        return self._penalty_pct
+
+
+class InvalidResponseStructureError(AGException):
     """A custom exception for the AutomatedGrader class."""
 
     def __init__(self, message):
         self.message = message
-        super().__init__(self.message)
+        super().__init__(self.message, penalty_pct=AGRubric.INVALID_RESPONSE_STRUCTURE_PENALTY_PCT)
 
 
-class IncorrectResponseValueError(Exception):
+class IncorrectResponseValueError(AGException):
     """A custom exception for the AutomatedGrader class."""
 
     def __init__(self, message):
         self.message = message
-        super().__init__(self.message)
+        super().__init__(self.message, penalty_pct=AGRubric.INCORRECT_RESPONSE_VALUE_PENALTY_PCT)
 
 
-class IncorrectResponseTypeError(Exception):
+class IncorrectResponseTypeError(AGException):
     """A custom exception for the AutomatedGrader class."""
 
     def __init__(self, message):
         self.message = message
-        super().__init__(self.message)
+        super().__init__(self.message, penalty_pct=AGRubric.INCORRECT_RESPONSE_TYPE_PENALTY_PCT)
 
 
-class ResponseFailedError(Exception):
+class ResponseFailedError(AGException):
     """A custom exception for the AutomatedGrader class."""
 
     def __init__(self, message):
         self.message = message
-        super().__init__(self.message)
+        super().__init__(self.message, penalty_pct=AGRubric.RESPONSE_FAILED_PENALTY_PCT)

grader/grader.py

@@ -4,14 +4,25 @@
 import json
 import os
 
+from pydantic import BaseModel, field_validator, model_validator
+
 from .exceptions import (
+    AGException,
     IncorrectResponseTypeError,
     IncorrectResponseValueError,
     InvalidResponseStructureError,
     ResponseFailedError,
 )
 
 
+VALID_MESSAGE_TYPES = [
+    "Success",
+    IncorrectResponseTypeError.__name__,
+    IncorrectResponseValueError.__name__,
+    InvalidResponseStructureError.__name__,
+    ResponseFailedError.__name__,
+]
+
 HERE = os.path.abspath(os.path.dirname(__file__))
 REQUIRED_KEYS_SPEC = "required-keys.json"
 REQUIRED_KEYS_PATH = os.path.join(HERE, "data", REQUIRED_KEYS_SPEC)
@@ -20,15 +31,59 @@
 AI_RESPONSE = {"content": "a response from the AI", "additional_kwargs": {}, "type": "ai", "example": False}
 
 
+class Grade(BaseModel):
+    """
+    This is the base class for all Grader types. It provides the common interface and
+    functionality for grading, but does not implement the grading logic itself.
+    Subclasses should override the necessary methods to provide the grading logic.
+    """
+
+    potential_points: float
+    grade: float
+    message: str
+    message_type: str
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.model_validate(self)
+
+    @model_validator(mode="after")
+    def validate_grade(self) -> "Grade":
+        """Validate that the grade is >= 0 and <= potential_points"""
+        if self.grade < 0:
+            raise ValueError(f"grade must be at least 0.00. received: {self.grade}")
+        if self.grade > self.potential_points:
+            raise ValueError(f"grade must be less than or equal to potential_points. received: {self.grade}")
+        return self
+
+    @field_validator("message_type")
+    def message_type_is_valid(self, message_type):
+        """Validate that the message_type is valid"""
+        if message_type not in VALID_MESSAGE_TYPES:
+            raise ValueError(f"message_type must be one of {VALID_MESSAGE_TYPES}")
+        return message_type
+
+
 # flake8: noqa: E701
 class AutomatedGrader:
     """Grade a submission against an assignment."""
 
-    def __init__(self, assignment):
-        self.assignment = assignment
+    def __init__(self, assignment, potential_points=100):
+        self._assignment = assignment
+        self._potential_points = potential_points
         with open(REQUIRED_KEYS_PATH, "r", encoding="utf-8") as f:  # pylint: disable=invalid-name
             self.required_keys = json.load(f)
 
+    @property
+    def assignment(self):
+        """Return the assignment."""
+        return self._assignment
+
+    @property
+    def potential_points(self):
+        """Return the potential points for the assignment."""
+        return self._potential_points
+
     def validate_keys(self, subject, control):
         """Validate that the subject has all the keys in the control dict."""
         assignment_keys = set(subject.keys())
@@ -154,26 +209,25 @@ def validate(self):
         self.validate_body()
         self.validate_metadata()
 
-    def grade_response(self, grade, message=None):
+    def grade_response(self, message: AGException = None):
         """Create a grade dict from the assignment."""
+        grade = self.potential_points * (1 - (message.penalty_pct if message else 0))
         message_type = message.__class__.__name__ if message else "Success"
         message = str(message) if message else "Great job!"
-        return {
-            "grade": grade,
-            "message": message,
-            "message_type": message_type,
-        }
+
+        grade = Grade(grade=grade, message=message, message_type=message_type, potential_points=self.potential_points)
+        return grade.model_dump()
 
     def grade(self):
         """Grade the assignment."""
         try:
             self.validate()
         except InvalidResponseStructureError as e:
-            return self.grade_response(70, e)
+            return self.grade_response(e)
         except ResponseFailedError as e:
-            return self.grade_response(80, e)
+            return self.grade_response(e)
         except IncorrectResponseValueError as e:
-            return self.grade_response(85, e)
+            return self.grade_response(e)
         except IncorrectResponseTypeError as e:
-            return self.grade_response(90, e)
-        return self.grade_response(100)
+            return self.grade_response(e)
+        return self.grade_response()

grader/tests/test_responses.py

@@ -23,7 +23,7 @@ def test_success(self):
         assert isinstance(grade, dict), "The grade is not a dictionary"
         assert grade["message_type"] == "Success"
         assert "grade" in grade, "The dictionary does not contain the key 'grade'"
-        assert isinstance(grade["grade"], int), "The grade is not an int"
+        assert isinstance(grade["grade"], float), "The grade is not an float"
         assert grade["grade"] == 100, "The grade is not 100"
 
         assert "message" in grade, "The dictionary does not contain the key 'message'"
@@ -39,7 +39,7 @@ def test_success_verbose(self):
         assert isinstance(grade, dict), "The grade is not a dictionary"
         assert grade["message_type"] == "Success"
         assert "grade" in grade, "The dictionary does not contain the key 'grade'"
-        assert isinstance(grade["grade"], int), "The grade is not an int"
+        assert isinstance(grade["grade"], float), "The grade is not an float"
         assert grade["grade"] == 100, "The grade is not 100"
 
         assert "message" in grade, "The dictionary does not contain the key 'message'"

requirements.txt

@@ -18,3 +18,6 @@ pydocstringformatter==0.7.3
 tox==4.11.3
 codespell==2.2.6
 python-dotenv==1.0.0
+
+# package dependencies
+pydantic==2.5.2

setup.py

@@ -15,5 +15,5 @@
     package_data={
         "automated_grader": ["*.md"],
     },
-    install_requires=[],
+    install_requires=["pydantic>=2.0.0"],
 )