feat(testers): add external testers (#38)

* add external testers

* fix lint

* add type annotations

checker/__main__.py

@@ -94,7 +94,8 @@ def check(
         deadlines_config=private_course_driver.get_deadlines_file_path(),
     )
     tester = Tester.create(
-        system=course_config.system,
+        root=root,
+        course_config=course_config,
         cleanup=not no_clean,
         dry_run=dry_run,
     )
@@ -157,7 +158,8 @@ def grade(
         deadlines_config=private_course_driver.get_deadlines_file_path(),
     )
     tester = Tester.create(
-        system=course_config.system,
+        root=execution_folder,
+        course_config=course_config,
     )
 
     grade_on_ci(

checker/course/config.py

@@ -48,6 +48,7 @@ class CourseConfig:
     # checker default
     layout: str = 'groups'
     executor: str = 'sandbox'
+    tester_path: str | None = None
 
     # info
     links: dict[str, str] | None = None

checker/testers/tester.py

@@ -5,13 +5,27 @@
 from abc import abstractmethod
 from dataclasses import dataclass
 from pathlib import Path
-from typing import Any
+from typing import Any, Dict
 
+from ..course import CourseConfig
 from ..exceptions import RunFailedError, TaskTesterTestConfigException, TesterNotImplemented
 from ..executors.sandbox import Sandbox
 from ..utils.print import print_info
 
 
+def _create_external_tester(tester_path: Path, dry_run: bool, cleanup: bool) -> Tester:
+    globls: Dict[str, Any] = {}
+    with open(tester_path) as f:
+        tester_code = compile(f.read(), tester_path.absolute(), 'exec')
+        exec(tester_code, globls)
+    tester_cls = globls.get('CustomTester')
+    if tester_cls is None:
+        raise TesterNotImplemented(f'class CustomTester not found in file {tester_path}')
+    if not issubclass(tester_cls, Tester):
+        raise TesterNotImplemented(f'class CustomTester in {tester_path} is not inherited from testers.Tester')
+    return tester_cls(dry_run=dry_run, cleanup=cleanup)
+
+
 class Tester:
     """Entrypoint to testing system
     Tester holds the course object and manage testing of single tasks,
@@ -27,7 +41,6 @@ class TaskTestConfig:
         """Task Tests Config
         Configure how task will copy files, check, execute and so on
         """
-        pass
 
         @classmethod
         def from_json(
@@ -75,7 +88,8 @@ def __init__(
     @classmethod
     def create(
             cls,
-            system: str,
+            root: Path,
+            course_config: CourseConfig,
             cleanup: bool = True,
             dry_run: bool = False,
     ) -> 'Tester':
@@ -87,6 +101,7 @@ def create(
         @param dry_run: Setup dry run mode (really executes nothing)
         @return: Configured Tester object (python, cpp, etc.)
         """
+        system = course_config.system
         if system == 'python':
             from . import python
             return python.PythonTester(cleanup=cleanup, dry_run=dry_run)
@@ -96,6 +111,11 @@ def create(
         elif system == 'cpp':
             from . import cpp
             return cpp.CppTester(cleanup=cleanup, dry_run=dry_run)
+        elif system == 'external':
+            path = course_config.tester_path
+            if path is None:
+                raise TesterNotImplemented('tester_path is not specified in course config')
+            return _create_external_tester(root / path, cleanup=cleanup, dry_run=dry_run)
         else:
             raise TesterNotImplemented(f'Tester for <{system}> are not supported right now')
 

tests/testers/test_tester.py

@@ -12,6 +12,27 @@
 from checker.testers.make import MakeTester
 from checker.testers.python import PythonTester
 from checker.testers.tester import Tester
+from checker.course import CourseConfig
+
+
+def create_test_course_config(**kwargs) -> CourseConfig:
+    return CourseConfig(
+        name='test',
+        deadlines='',
+        templates='',
+        manytask_url='',
+        course_group='',
+        public_repo='',
+        students_group='',
+        **kwargs,
+    )
+
+def write_tester_to_file(path: Path, content: str) -> Path:
+    filename = path / 'tester.py'
+    content = inspect.cleandoc(content)
+    with open(filename, 'w') as f:
+        f.write(content)
+    return filename
 
 
 class TestTester:
@@ -21,12 +42,45 @@ class TestTester:
         ('make', MakeTester),
     ])
     def test_right_tester_created(self, tester_name: str, tester_class: Type[Tester]) -> None:
-        tester = Tester.create(tester_name)
+        course_config = create_test_course_config(system=tester_name)
+        tester = Tester.create(root=Path(), course_config=course_config)
         assert isinstance(tester, tester_class)
 
+    def test_external_tester(self, tmp_path: Path):
+        TESTER = """
+        from checker.testers import Tester
+        class CustomTester(Tester):
+            definitely_external_tester = 'Yes!'
+        """
+        course_config = create_test_course_config(system='external', tester_path='tester.py')
+        write_tester_to_file(tmp_path, TESTER)
+        tester = Tester.create(root=tmp_path, course_config=course_config)
+        assert hasattr(tester, 'definitely_external_tester')
+
+    NOT_A_TESTER = """
+        class NotATester:
+            definitely_external_tester = 'Yes!'
+        """
+
+    NOT_INHERITED_TESTER = """
+        class CustomTester:
+            definitely_external_tester = 'Yes!'
+        """
+
+    @pytest.mark.parametrize('tester_content', [
+        NOT_A_TESTER,
+        NOT_INHERITED_TESTER,
+    ])
+    def test_invalid_external_tester(self, tmp_path: Path, tester_content):
+        course_config = create_test_course_config(system='external', tester_path='tester.py')
+        write_tester_to_file(tmp_path, tester_content)
+        with pytest.raises(TesterNotImplemented):
+            Tester.create(root=tmp_path, course_config=course_config)
+
     def test_wrong_tester(self) -> None:
+        course_config = create_test_course_config(system='definitely-wrong-tester')
         with pytest.raises(TesterNotImplemented):
-            Tester.create('definitely-wrong-tester')
+            Tester.create(root=Path(), course_config=course_config)
 
 
 @dataclass