From bb9eb45d97e62d8165f1dc457d98a066203f8859 Mon Sep 17 00:00:00 2001 From: Andrew Bogle Date: Thu, 20 Feb 2025 11:10:24 -0500 Subject: [PATCH 1/7] prevent model not found error after using da wizard --- agentstack/cli/cli.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agentstack/cli/cli.py b/agentstack/cli/cli.py index 70e7c3ba..70fc5ab6 100644 --- a/agentstack/cli/cli.py +++ b/agentstack/cli/cli.py @@ -16,7 +16,7 @@ 'deepseek/deepseek-coder', 'deepseek/deepseek-reasoner', 'openai/gpt-4o', - 'anthropic/claude-3-5-sonnet', + 'anthropic/claude-3-5-sonnet-20240620', 'openai/o1-preview', 'openai/gpt-4-turbo', 'anthropic/claude-3-opus', From 536219e06bdfae939ef21a3f3f02bf7daf8bca52 Mon Sep 17 00:00:00 2001 From: Andrew Bogle Date: Thu, 20 Feb 2025 13:22:12 -0500 Subject: [PATCH 2/7] minor copy updates --- CONTRIBUTING.md | 39 ++++++++++++++++++++++++++++++--------- 1 file changed, 30 insertions(+), 9 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 85853997..706b0f5a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,12 +1,14 @@ # Contributing -First of all, __thank you__ for your interest in contributing to AgentStack! Even the smallest contributions help a _ton_. + +First of all, **thank you** for your interest in contributing to AgentStack! Even the smallest contributions help a _ton_. Our vision is to build the defacto CLI for quickly spinning up an AI Agent project. We want to be the [create-react-app](https://create-react-app.dev/) of agents. Our inspiration also includes the oh-so-convenient [Angular CLI](https://v17.angular.io/cli). ### Exclusive Contributor Sticker + AgentStack contributors all receive a free sticker pack including an exclusive holographic sticker only available to contributors to the project :) -Once your PR is merge, fill out [this form](https://docs.google.com/forms/d/e/1FAIpQLSfvBEnsT8nsQleonJHoWQtHuhbsgUJ0a9IjOqeZbMGkga2NtA/viewform?usp=sf_link) and I'll send your sticker pack out ASAP! <3 +Once your PR is merged, fill out [this form](https://docs.google.com/forms/d/e/1FAIpQLSfvBEnsT8nsQleonJHoWQtHuhbsgUJ0a9IjOqeZbMGkga2NtA/viewform?usp=sf_link) and I'll send your sticker pack out ASAP! <3 ## How to Help @@ -16,70 +18,89 @@ The best place to engage in conversation about your contribution is in the Issue ## Setup -1. Clone the repo - `git clone https://github.com/AgentOps-AI/AgentStack.git` +1. Fork the repo from the github website or with [gh repo fork AgentOps-AI/AgentStack](https://cli.github.com/manual/gh_repo_fork) + +2. Clone the forked repo and get in there! + `ssh example` + `git clone git@github.com:/AgentStack.git` `cd AgentStack` -2. Install agentstack as an edtiable project and set it up for development and testing + +3. Install agentstack as an edtiable project and set it up for development and testing `pip install -e .[dev,test]` This will install the CLI locally and in editable mode so you can use `agentstack ` to test your latest changes - ## Adding Tools + If you're reading this section, you probably have a product that AI agents can use as a tool. We're glad you're here! Adding tools is easy once you understand the project structure. Our documentation for adding tools is available on our hosted docs [here](https://docs.agentstack.sh/contributing/adding-tools). ## Before creating your PR + Be sure that you are opening a PR using a branch other than `main` on your fork. This enables us to pull your branch and make modifications to the PR with your permission that may be helpful. ### Formatting + AgentStack uses Ruff formatter for consistent code formatting. To format your code, run: + ```bash pip install ruff ruff format . ``` ### Type Checking + AgentStack uses MyPy for type checking. To check types, run: + ```bash mypy agentstack ``` ### Pre-Commit Hooks + Ruff and MyPy can be run as pre-commit hooks. To enable these hooks, run: + ```bash pre-commit install ``` ## Tests -CLI tests are a bit hacky, so we are not tracking coverage. + +CLI tests are a bit hacky, so we are not tracking coverage. That said, _some_ testing is required for any new functionality added by a PR. Tests MUST pass to have your PR merged. We _will not_ allow main to be in a failing state, so if your tests are failing, this is your problem to fix. ### Run tests locally + Install the testing requirements + ```bash pip install 'agentstack[test]' ``` Then run tests in all supported python versions with + ```bash tox ``` ## Need Help? -If you're reading this, we're very thankful you wanted to contribute! I understand it can be a little overwhelming to + +If you're reading this, we're very thankful you wanted to contribute! I understand it can be a little overwhelming to get up to speed on a project like this and we are here to help! ### Open a draft PR + While we can't promise to write code for you, if you're stuck or need advice/help, open a draft PR and explain what you were trying to build and where you're stuck! Chances are, one of us have the context needed to help you get unstuck :) ### Chat on our Discord + We have an active [Discord server](https://discord.gg/JdWkh9tgTQ) with contributors and AgentStack users! There is a channel just for contributors on there. Feel free to drop a message explaining what you're trying to build and why you're stuck. Someone from our team should reply soon! # Thank You! + The team behind AgentStack believe that the barrier to entry for building agents is far too high right now! We believe that this technology can be streamlined and made more accessible. If you're here, you likely feel the same! Any contribution is appreciated. -If you're looking for work, we are _always_ open to hiring passionate engineers of all skill levels! While closing issues cannot guarantee an offer, we've found that engineers who contribute to our open source repo are some of the best we could ever hope to find via recruiters! Be active in the community and let us know you're interested in joining the team! \ No newline at end of file +If you're looking for work, we are _always_ open to hiring passionate engineers of all skill levels! While closing issues cannot guarantee an offer, we've found that engineers who contribute to our open source repo are some of the best we could ever hope to find via recruiters! Be active in the community and let us know you're interested in joining the team! From ad7371b9656ec2afcd5435e5c553ed6a7c63b047 Mon Sep 17 00:00:00 2001 From: Andrew Bogle Date: Sat, 22 Feb 2025 14:53:31 -0500 Subject: [PATCH 3/7] undo auto-formatting of .md file --- CONTRIBUTING.md | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 706b0f5a..5639bddc 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,11 +1,9 @@ # Contributing - -First of all, **thank you** for your interest in contributing to AgentStack! Even the smallest contributions help a _ton_. +First of all, __thank you__ for your interest in contributing to AgentStack! Even the smallest contributions help a _ton_. Our vision is to build the defacto CLI for quickly spinning up an AI Agent project. We want to be the [create-react-app](https://create-react-app.dev/) of agents. Our inspiration also includes the oh-so-convenient [Angular CLI](https://v17.angular.io/cli). ### Exclusive Contributor Sticker - AgentStack contributors all receive a free sticker pack including an exclusive holographic sticker only available to contributors to the project :) Once your PR is merged, fill out [this form](https://docs.google.com/forms/d/e/1FAIpQLSfvBEnsT8nsQleonJHoWQtHuhbsgUJ0a9IjOqeZbMGkga2NtA/viewform?usp=sf_link) and I'll send your sticker pack out ASAP! <3 @@ -19,88 +17,70 @@ The best place to engage in conversation about your contribution is in the Issue ## Setup 1. Fork the repo from the github website or with [gh repo fork AgentOps-AI/AgentStack](https://cli.github.com/manual/gh_repo_fork) - 2. Clone the forked repo and get in there! `ssh example` `git clone git@github.com:/AgentStack.git` `cd AgentStack` - 3. Install agentstack as an edtiable project and set it up for development and testing `pip install -e .[dev,test]` This will install the CLI locally and in editable mode so you can use `agentstack ` to test your latest changes ## Adding Tools - If you're reading this section, you probably have a product that AI agents can use as a tool. We're glad you're here! Adding tools is easy once you understand the project structure. Our documentation for adding tools is available on our hosted docs [here](https://docs.agentstack.sh/contributing/adding-tools). ## Before creating your PR - Be sure that you are opening a PR using a branch other than `main` on your fork. This enables us to pull your branch and make modifications to the PR with your permission that may be helpful. ### Formatting - AgentStack uses Ruff formatter for consistent code formatting. To format your code, run: - ```bash pip install ruff ruff format . ``` ### Type Checking - AgentStack uses MyPy for type checking. To check types, run: - ```bash mypy agentstack ``` ### Pre-Commit Hooks - Ruff and MyPy can be run as pre-commit hooks. To enable these hooks, run: - ```bash pre-commit install ``` ## Tests - -CLI tests are a bit hacky, so we are not tracking coverage. +CLI tests are a bit hacky, so we are not tracking coverage. That said, _some_ testing is required for any new functionality added by a PR. Tests MUST pass to have your PR merged. We _will not_ allow main to be in a failing state, so if your tests are failing, this is your problem to fix. ### Run tests locally - Install the testing requirements - ```bash pip install 'agentstack[test]' ``` Then run tests in all supported python versions with - ```bash tox ``` ## Need Help? - -If you're reading this, we're very thankful you wanted to contribute! I understand it can be a little overwhelming to +If you're reading this, we're very thankful you wanted to contribute! I understand it can be a little overwhelming to get up to speed on a project like this and we are here to help! ### Open a draft PR - While we can't promise to write code for you, if you're stuck or need advice/help, open a draft PR and explain what you were trying to build and where you're stuck! Chances are, one of us have the context needed to help you get unstuck :) ### Chat on our Discord - We have an active [Discord server](https://discord.gg/JdWkh9tgTQ) with contributors and AgentStack users! There is a channel just for contributors on there. Feel free to drop a message explaining what you're trying to build and why you're stuck. Someone from our team should reply soon! # Thank You! - The team behind AgentStack believe that the barrier to entry for building agents is far too high right now! We believe that this technology can be streamlined and made more accessible. If you're here, you likely feel the same! Any contribution is appreciated. -If you're looking for work, we are _always_ open to hiring passionate engineers of all skill levels! While closing issues cannot guarantee an offer, we've found that engineers who contribute to our open source repo are some of the best we could ever hope to find via recruiters! Be active in the community and let us know you're interested in joining the team! +If you're looking for work, we are _always_ open to hiring passionate engineers of all skill levels! While closing issues cannot guarantee an offer, we've found that engineers who contribute to our open source repo are some of the best we could ever hope to find via recruiters! Be active in the community and let us know you're interested in joining the team! \ No newline at end of file From 06c42a92776d03ceafe5cc09eccf7616d581c276 Mon Sep 17 00:00:00 2001 From: Andrew Bogle Date: Tue, 4 Mar 2025 13:50:05 -0500 Subject: [PATCH 4/7] fix the anthropic models and add a test --- agentstack/cli/cli.py | 8 ++--- tests/test_preferred_models.py | 60 ++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 tests/test_preferred_models.py diff --git a/agentstack/cli/cli.py b/agentstack/cli/cli.py index 70fc5ab6..2c08ca19 100644 --- a/agentstack/cli/cli.py +++ b/agentstack/cli/cli.py @@ -1,5 +1,4 @@ from typing import Optional -import os, sys from art import text2art import inquirer from agentstack import conf, log @@ -16,10 +15,10 @@ 'deepseek/deepseek-coder', 'deepseek/deepseek-reasoner', 'openai/gpt-4o', - 'anthropic/claude-3-5-sonnet-20240620', 'openai/o1-preview', 'openai/gpt-4-turbo', - 'anthropic/claude-3-opus', + 'anthropic/claude-3-opus-latest', + 'anthropic/claude-3-5-sonnet-20240620', ] @@ -38,7 +37,7 @@ def welcome_message(): def undo() -> None: """Undo the last committed changes.""" conf.assert_project() - + changed_files = repo.get_uncommitted_files() if changed_files: log.warning("There are uncommitted changes that may be overwritten.") @@ -113,4 +112,3 @@ def parse_insertion_point(position: Optional[str] = None) -> Optional[InsertionP raise ValueError(f"Position must be one of {','.join(valid_positions)}.") return next(x for x in InsertionPoint if x.value == position) - diff --git a/tests/test_preferred_models.py b/tests/test_preferred_models.py new file mode 100644 index 00000000..ce1c540d --- /dev/null +++ b/tests/test_preferred_models.py @@ -0,0 +1,60 @@ +from litellm import models_by_provider +from agentstack.cli.cli import PREFERRED_MODELS +from difflib import get_close_matches + + +def clean_model_name(provider: str, model: str) -> str: + """ + Clean up model name by removing duplicate provider strings. + Seems like in litellm groq and deepseek have the provider twice in the model name. + """ + if model.startswith(f"{provider}/"): + return f"{provider}/{model[len(provider) + 1 :]}" + return f"{provider}/{model}" + + +def find_similar_models(model: str, all_models: set, num_suggestions: int = 3) -> list[str]: + """ + Find similar model names using string matching. + If the test fails, now you can see the ideal model to replace a broken one with. + """ + try: + provider, model_name = model.split('/') + except ValueError: + return get_close_matches(model, all_models, n=num_suggestions, cutoff=0.3) + + provider_models = [m for m in all_models if m.startswith(f"{provider}/")] + if provider_models: + matches = get_close_matches(model, provider_models, n=num_suggestions, cutoff=0.3) + if matches: + return matches + print("https://docs.litellm.ai/docs/providers contains supported models for each provider.") + return get_close_matches(model, all_models, n=num_suggestions, cutoff=0.3) + + +def test_preferred_models_validity(): + """Test that all PREFERRED_MODELS are valid LiteLLM models.""" + all_litellm_models = set() + for provider, models in models_by_provider.items(): + for model in models: + full_model_name = clean_model_name(provider, model) + all_litellm_models.add(full_model_name) + + invalid_models_with_suggestions = {} + for model in PREFERRED_MODELS: + if model not in all_litellm_models: + suggestions = find_similar_models(model, all_litellm_models) + invalid_models_with_suggestions[model] = suggestions + + if invalid_models_with_suggestions: + error_message = "The following models are not in LiteLLM's supported models:\n" + for model, suggestions in invalid_models_with_suggestions.items(): + error_message += f"\n- {model}" + if suggestions: + error_message += "\n Similar available models:" + for suggestion in suggestions: + error_message += f"\n * {suggestion}" + else: + error_message += "\n No similar models found." + + assert False, error_message From 13f1b203360f10ffd0d3cfa688d7f84af84bbe3d Mon Sep 17 00:00:00 2001 From: Andrew Bogle Date: Tue, 4 Mar 2025 14:06:56 -0500 Subject: [PATCH 5/7] slightly better console messaging --- tests/test_preferred_models.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/tests/test_preferred_models.py b/tests/test_preferred_models.py index ce1c540d..218453a3 100644 --- a/tests/test_preferred_models.py +++ b/tests/test_preferred_models.py @@ -28,7 +28,6 @@ def find_similar_models(model: str, all_models: set, num_suggestions: int = 3) - matches = get_close_matches(model, provider_models, n=num_suggestions, cutoff=0.3) if matches: return matches - print("https://docs.litellm.ai/docs/providers contains supported models for each provider.") return get_close_matches(model, all_models, n=num_suggestions, cutoff=0.3) @@ -47,7 +46,10 @@ def test_preferred_models_validity(): invalid_models_with_suggestions[model] = suggestions if invalid_models_with_suggestions: - error_message = "The following models are not in LiteLLM's supported models:\n" + error_message = ( + "The following models are not in LiteLLM's supported models:\n" + "\nFor a complete list of supported models, visit: https://docs.litellm.ai/docs/providers\n" + ) for model, suggestions in invalid_models_with_suggestions.items(): error_message += f"\n- {model}" if suggestions: From 01f661688da3c7618157d7f8088ae21980c5f8c4 Mon Sep 17 00:00:00 2001 From: Andrew Bogle Date: Tue, 4 Mar 2025 23:23:15 -0500 Subject: [PATCH 6/7] i wonder when I installed litellm in my venv manually 0,0 --- tests/test_preferred_models.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_preferred_models.py b/tests/test_preferred_models.py index 218453a3..e89df3af 100644 --- a/tests/test_preferred_models.py +++ b/tests/test_preferred_models.py @@ -1,6 +1,16 @@ -from litellm import models_by_provider -from agentstack.cli.cli import PREFERRED_MODELS +from importlib.util import find_spec +import pytest +import subprocess +import sys from difflib import get_close_matches +from agentstack.cli.cli import PREFERRED_MODELS + + +@pytest.fixture(scope="session", autouse=True) +def install_litellm(): + """Install litellm if not already installed.""" + if find_spec("litellm") is not None: + subprocess.check_call([sys.executable, "-m", "pip", "install", "litellm"]) def clean_model_name(provider: str, model: str) -> str: @@ -33,6 +43,8 @@ def find_similar_models(model: str, all_models: set, num_suggestions: int = 3) - def test_preferred_models_validity(): """Test that all PREFERRED_MODELS are valid LiteLLM models.""" + from litellm import models_by_provider + all_litellm_models = set() for provider, models in models_by_provider.items(): for model in models: From 11d51e54bed112cd2ce430079bd07e25e535ab7a Mon Sep 17 00:00:00 2001 From: Andrew Bogle Date: Thu, 6 Mar 2025 19:26:23 -0500 Subject: [PATCH 7/7] fix my flu-induced inverted conditional and add a couple logs --- tests/test_preferred_models.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_preferred_models.py b/tests/test_preferred_models.py index e89df3af..d0b8bee6 100644 --- a/tests/test_preferred_models.py +++ b/tests/test_preferred_models.py @@ -9,8 +9,12 @@ @pytest.fixture(scope="session", autouse=True) def install_litellm(): """Install litellm if not already installed.""" - if find_spec("litellm") is not None: + print("\nChecking for litellm installation...") + spec = find_spec("litellm") + if spec is None: + print("litellm not found, installing...") subprocess.check_call([sys.executable, "-m", "pip", "install", "litellm"]) + print("litellm installation complete") def clean_model_name(provider: str, model: str) -> str: