Create separate test environments for each framework.

.github/workflows/python-testing.yml

@@ -25,15 +25,17 @@ jobs:
 
     strategy:
       matrix:
-        python-version: [3.10,3.11,3.12]
+        python-version: ["3.10","3.11","3.12"]  # versions need to be strings
 
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python
-        uses: actions/setup-python@v2
+        uses: actions/setup-python@v5
         with:
-          python-version: '3.11' # Use a default Python version for running tox
+          python-version: ${{ matrix.python-version }}
       - name: Install tox
         run: pip install tox
+      - name: Install tox-gh-actions
+        run: pip install tox-gh-actions  # allows tox to get the current python version
       - name: Run tests with tox
-        run: tox
+        run: tox  # we can't pass arguments to tox when using tox-gh-actions, so all frameworks happen in one run

pyproject.toml

@@ -46,14 +46,12 @@ test = [
 crewai = [
     "crewai==0.100.0",
     "crewai-tools==0.33.0",
+    "shapely==2.0.6"  # crewai-tools needs this, but 2.0.7 is broken
 ]
 langgraph = [
     "langgraph>=0.2.61",
     "langchain>=0.3.14",
 ]
-all = [
-    "agentstack[dev,test,crewai,langgraph]",
-]
 
 [tool.setuptools.package-data]
 agentstack = ["templates/**/*"]

tests/test_cli_templates.py

@@ -11,7 +11,8 @@
 
 class CLITemplatesTest(unittest.TestCase):
     def setUp(self):
-        self.project_dir = Path(BASE_PATH / 'tmp/cli_templates')
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'cli_templates'
         os.makedirs(self.project_dir, exist_ok=True)
         os.chdir(self.project_dir)
 

tests/test_cli_tools.py

@@ -17,7 +17,8 @@
 # TODO parameterized framework
 class CLIToolsTest(unittest.TestCase):
     def setUp(self):
-        self.project_dir = Path(BASE_PATH / 'tmp/cli_tools')
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'cli_tools'
         os.makedirs(self.project_dir, exist_ok=True)
         os.chdir(self.project_dir)
 

tests/test_frameworks.py

@@ -3,7 +3,7 @@
 from pathlib import Path
 import shutil
 import unittest
-from parameterized import parameterized, parameterized_class
+from parameterized import parameterized
 
 from agentstack.conf import ConfigFile, set_path
 from agentstack.exceptions import ValidationError
@@ -16,9 +16,9 @@
 BASE_PATH = Path(__file__).parent
 
 
-@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS])
 class TestFrameworks(unittest.TestCase):
     def setUp(self):
+        self.framework = os.getenv('TEST_FRAMEWORK')
         self.project_dir = BASE_PATH / 'tmp' / self.framework / 'test_frameworks'
 
         os.makedirs(self.project_dir)

tests/test_frameworks_langgraph.py

@@ -18,7 +18,8 @@
 
 class FrameworksLanggraphTest(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / 'tmp/frameworks/langgraph'
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'langgraph'
         conf.set_path(self.project_dir)
         os.makedirs(self.project_dir / 'src/config')
 

tests/test_generation_agent.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 import shutil
 import unittest
-from parameterized import parameterized_class
 import ast
 
 from agentstack.conf import ConfigFile, set_path
@@ -15,10 +14,10 @@
 BASE_PATH = Path(__file__).parent
 
 
-@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS])
 class TestGenerationAgent(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / 'tmp' / 'agent_generation'
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'agent_generation'
 
         os.makedirs(self.project_dir)
         os.makedirs(self.project_dir / 'src')

tests/test_generation_files.py

@@ -18,7 +18,8 @@
 # TODO copy files to working directory
 class GenerationFilesTest(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / "tmp" / "generation_files"
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'generation_files'
         os.makedirs(self.project_dir)
 
         shutil.copy(BASE_PATH / "fixtures/agentstack.json", self.project_dir / "agentstack.json")

tests/test_generation_tasks.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 import shutil
 import unittest
-from parameterized import parameterized_class
 import ast
 
 from agentstack.conf import ConfigFile, set_path
@@ -16,10 +15,10 @@
 BASE_PATH = Path(__file__).parent
 
 
-@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS])
 class TestGenerationAgent(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / 'tmp' / 'agent_generation'
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'agent_generation'
 
         os.makedirs(self.project_dir)
         os.makedirs(self.project_dir / 'src')

tests/test_generation_tool.py

@@ -15,10 +15,10 @@
 
 
 # TODO parameterize all tools
-@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS])
 class TestGenerationTool(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / 'tmp' / 'tool_generation'
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'tool_generation'
 
         os.makedirs(self.project_dir)
         os.makedirs(self.project_dir / 'src')

tests/test_inputs_config.py

@@ -11,7 +11,8 @@
 
 class InputsConfigTest(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / "tmp/inputs_config"
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'inputs_config'
         os.makedirs(self.project_dir)
         os.makedirs(self.project_dir / "src/config")
 

tests/test_log.py

@@ -1,5 +1,5 @@
 import unittest
-import sys
+import os, sys
 import io
 import logging
 import shutil
@@ -12,8 +12,8 @@
 
 class TestLog(unittest.TestCase):
     def setUp(self):
-        # Create test directory if it doesn't exist
-        self.test_dir = BASE_PATH / 'tmp/test_log'
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.test_dir = BASE_PATH / 'tmp' / self.framework / 'test_log'
         self.test_dir.mkdir(parents=True, exist_ok=True)
 
         # Set log file to test directory

tests/test_project_run.py

@@ -2,7 +2,6 @@
 from pathlib import Path
 import shutil
 import unittest
-from parameterized import parameterized_class
 
 from agentstack import conf
 from agentstack.conf import ConfigFile
@@ -12,10 +11,10 @@
 BASE_PATH = Path(__file__).parent
 
 
-@parameterized_class([{"framework": framework} for framework in frameworks.SUPPORTED_FRAMEWORKS])
 class ProjectRunTest(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / 'tmp/project_run' / self.framework
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'project_run'
 
         os.makedirs(self.project_dir)
         os.makedirs(self.project_dir / 'src')

tests/test_tasks_config.py

@@ -13,7 +13,8 @@
 
 class AgentConfigTest(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / 'tmp/task_config'
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'task_config'
         os.makedirs(self.project_dir / 'src/config')
         conf.set_path(self.project_dir)
 

tests/test_templates_config.py

@@ -21,7 +21,8 @@
 
 class TemplateConfigTest(unittest.TestCase):
     def setUp(self):
-        self.project_dir = BASE_PATH / 'tmp/template_config'
+        self.framework = os.getenv('TEST_FRAMEWORK')
+        self.project_dir = BASE_PATH / 'tmp' / self.framework / 'template_config'
         os.makedirs(self.project_dir, exist_ok=True)
 
     def tearDown(self):

tox.ini

@@ -1,30 +1,56 @@
-# tox (https://tox.readthedocs.io/) is a tool for running tests
-# in multiple virtualenvs. This configuration file will run the
-# test suite on all supported python versions. To use it, "pip install tox"
-# and then run "tox" from this directory.
+# Adding a New Framework
+# modify `envlist`, add extras to install optional-dependencies, and add a new 
+# `TEST_FRAMEWORK` section to `setenv`.
+
+# Developers
+# run just 3.12 tests with `tox -epy312-crewai,py312-langgraph,report`
+# TODO I have not found a way to alias this, yet. 
+
+# Coverage
+# codecov is configured to run on all frameworks and then be combined at the end. 
 
 [tox]
-envlist = py310,py311,py312
+envlist = py{310,311,312}-{crewai,langgraph}
+
+[gh-actions]
+# converts python versions to tox envlist values for github actions
+python =
+    3.10: py310
+    3.11: py311
+    3.12: py312
 
 [testenv]
-extras = all  # install dependencies for all frameworks, too
+extras = 
+    # framework specific dependencies
+    crewai: crewai  # installs agentstack[crewai]
+    langgraph: langgraph
 deps =
     pytest
     parameterized
     coverage
     mypy: mypy
 commands =
     coverage run --source=. --omit="**/tmp/**,./tmp/**/*" -m pytest -v {posargs}
-    coverage report -m
-    coverage xml
     mypy: mypy agentstack
 setenv =
     AGENTSTACK_IS_TEST_ENV = 1
     AGENTSTACK_TELEMETRY_OPT_OUT = 1
     AGENTSTACK_UPDATE_DISABLE = 1
+    # environment variables to determine active framework inside tests
+    crewai: TEST_FRAMEWORK = crewai
+    langgraph: TEST_FRAMEWORK = langgraph
+
+[testenv:report]
+deps = coverage
+skip_install = true
+commands =
+    coverage combine
+    coverage report -m
+    coverage xml
 
 [coverage:run]
 branch = True
+parallel = True
 source = .
 omit =
     **/tmp/**
@@ -40,4 +66,5 @@ exclude_lines =
     pragma: no cover
     ^\s*\.\.\.\s*$
     pass
-    if TYPE_CHECKING:
+    if TYPE_CHECKING:
+