From 35761c35f86774b9bad4987afe4dfe31de220d58 Mon Sep 17 00:00:00 2001 From: Ary Date: Tue, 26 Mar 2024 02:15:08 +0800 Subject: [PATCH 1/7] add solar model --- agent/language_models.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/agent/language_models.py b/agent/language_models.py index d47608f..52d3eea 100644 --- a/agent/language_models.py +++ b/agent/language_models.py @@ -23,6 +23,10 @@ def get_async_client(provider: str) -> AsyncOpenAI: return AsyncOpenAI( base_url="https://api.mistral.ai/v1/", api_key=os.getenv("MISTRAL_API_KEY") ) + if provider == "solar": + return AsyncOpenAI( + base_url="https://api.upstage.ai/v1/solar", api_key=os.getenv("SOLAR_API_KEY") + ) if provider == "ollama": return AsyncOpenAI(base_url="http://localhost:11434/v1/") raise NotImplementedError(f"Provider {provider} is not supported.") @@ -35,6 +39,10 @@ def get_sync_client(provider: str) -> OpenAI: return OpenAI( base_url="https://api.mistral.ai/v1/", api_key=os.getenv("MISTRAL_API_KEY") ) + if provider == "solar": + return OpenAI( + base_url="https://api.upstage.ai/v1/solar", api_key=os.getenv("SOLAR_API_KEY") + ) if provider == "ollama": return OpenAI(base_url="http://localhost:11434/v1/") if provider == "groq": From 52197fcb576d2d1f34e138fadda4949005e807c5 Mon Sep 17 00:00:00 2001 From: Ary Date: Tue, 2 Apr 2024 21:55:03 +0800 Subject: [PATCH 2/7] add solar --- agent/actions.py | 1 + agent/config.py | 15 +++++++++------ agent/language_models.py | 1 + eval/game.py | 26 ++++++++++++++++---------- script.py | 2 +- 5 files changed, 28 insertions(+), 17 deletions(-) diff --git a/agent/actions.py b/agent/actions.py index d0e31e6..8b24ab0 100644 --- a/agent/actions.py +++ b/agent/actions.py @@ -32,6 +32,7 @@ def call_llm( """ # Get the correct provider, default is mistral provider_name, model_name = get_provider_and_model(model) + print("Provider", provider_name, model_name) client = get_sync_client(provider_name) # Generate the prompts diff --git a/agent/config.py b/agent/config.py index 1fc65d2..86ce8d8 100644 --- a/agent/config.py +++ b/agent/config.py @@ -1,15 +1,18 @@ MODELS = { "OPENAI": { - "openai:gpt-4-0125-preview", - "openai:gpt-4", + # "openai:gpt-4-0125-preview", + # "openai:gpt-4", "openai:gpt-3.5-turbo-0125", # "openai:gpt-3.5-turbo-instruct", # not a chat model }, "MISTRAL": { - "mistral:mistral-small-latest", - "mistral:mistral-medium-latest", - "mistral:mistral-large-latest", - # "groq:mistral-8x6b-32768", + # "mistral:mistral-small-latest", + # "mistral:mistral-medium-latest", + # "mistral:mistral-large-latest", + "mistral:open-mistral-7b", + }, + "Solar": { + "solar:solar-1-mini-chat", }, } diff --git a/agent/language_models.py b/agent/language_models.py index 52d3eea..2dcf8f7 100644 --- a/agent/language_models.py +++ b/agent/language_models.py @@ -40,6 +40,7 @@ def get_sync_client(provider: str) -> OpenAI: base_url="https://api.mistral.ai/v1/", api_key=os.getenv("MISTRAL_API_KEY") ) if provider == "solar": + print("solar key", os.getenv("SOLAR_API_KEY")) return OpenAI( base_url="https://api.upstage.ai/v1/solar", api_key=os.getenv("SOLAR_API_KEY") ) diff --git a/eval/game.py b/eval/game.py index c36004f..bf70f53 100644 --- a/eval/game.py +++ b/eval/game.py @@ -18,7 +18,7 @@ from agent.config import MODELS -def generate_model(openai: bool = False, mistral: bool = True): +def generate_model(openai: bool = False, mistral: bool = True, solar: bool = False): models_available = [] for model, models in MODELS.items(): @@ -26,7 +26,9 @@ def generate_model(openai: bool = False, mistral: bool = True): models_available.extend(models) if mistral and model == "MISTRAL": models_available.extend(models) - + if solar and model == "Solar": + models_available.extend(models) + print('models:', models_available) random.seed() # Generate a pair of random two models random_model = random.choice(models_available) @@ -49,14 +51,15 @@ def __init__( model: Optional[str] = None, openai: bool = False, mistral: bool = True, + solar: bool = False, ): self.nickname = nickname - self.model = model or generate_model(openai=openai, mistral=mistral) + self.model = model or generate_model(openai=False, mistral=True, solar=False) self.openai = False self.mistral = True self.robot = Robot( action_space=None, - character="Ken", + character="Mistral", side=0, character_color=KEN_RED, ennemy_color=KEN_GREEN, @@ -74,12 +77,13 @@ def __init__( model: Optional[str] = None, openai: bool = False, mistral: bool = True, + solar: bool = False, ): self.nickname = nickname - self.model = model or generate_model(openai=openai, mistral=mistral) + self.model = model or generate_model(openai=False, mistral=False, solar=solar) self.robot = Robot( action_space=None, - character="Ken", + character="Solar", side=1, character_color=KEN_GREEN, ennemy_color=KEN_RED, @@ -120,7 +124,7 @@ class Game: render: Optional[bool] = False splash_screen: Optional[bool] = False save_game: Optional[bool] = False - characters: Optional[List[str]] = ["Ken", "Ken"] + characters: Optional[List[str]] = ["Makoto", "Sean"] outfits: Optional[List[int]] = [1, 3] frame_shape: Optional[List[int]] = [0, 0, 0] seed: Optional[int] = 42 @@ -136,7 +140,7 @@ def __init__( render: bool = False, save_game: bool = False, splash_screen: bool = False, - characters: List[str] = ["Ken", "Ken"], + characters: List[str] = ["Makoto", "Sean"], super_arts: List[int] = [3,3], outfits: List[int] = [1, 3], frame_shape: List[int] = [0, 0, 0], @@ -145,6 +149,7 @@ def __init__( player_2: Player2 = None, openai: bool = False, mistral: bool = True, + solar: bool = False, ): """_summary_ @@ -169,17 +174,18 @@ def __init__( self.observation, self.info = self.env.reset(seed=self.seed) self.openai = openai self.mistral = mistral + self.solar = solar self.player_1 = ( player_1 if player_1 - else Player1(nickname="Player 1", openai=self.openai, mistral=self.mistral) + else Player1(nickname="Player 1", openai=self.openai, mistral=self.mistral, solar=self.solar) # else Player1(nickname="Player 1", model="grok:mixtral-8x7b-32768") ) self.player_2 = ( player_2 if player_2 # else Player2(nickname="Player 2", model="openai:gpt-4-turbo-preview") - else Player2(nickname="Player 2", openai=self.openai, mistral=self.mistral) + else Player2(nickname="Player 2", openai=self.openai, mistral=self.mistral, solar=self.solar) ) def _init_settings(self) -> EnvironmentSettingsMultiAgent: diff --git a/script.py b/script.py index 5b653dd..a21c95a 100644 --- a/script.py +++ b/script.py @@ -15,7 +15,7 @@ def main(): # Environment Settings # Environment Settings while True: - game = Game(render=True, save_game=True, openai=True) + game = Game(render=True, save_game=True, openai=True, solar=True, mistral=False) game.run() From 92910cf9027954ae8903be94c1086a6056a9a8a8 Mon Sep 17 00:00:00 2001 From: Ary Date: Wed, 3 Apr 2024 03:32:11 +0800 Subject: [PATCH 3/7] add solar --- agent/config.py | 5 +++-- agent/language_models.py | 1 - eval/game.py | 15 +++++++++------ 3 files changed, 12 insertions(+), 9 deletions(-) diff --git a/agent/config.py b/agent/config.py index 86ce8d8..fc5a1e8 100644 --- a/agent/config.py +++ b/agent/config.py @@ -6,10 +6,11 @@ # "openai:gpt-3.5-turbo-instruct", # not a chat model }, "MISTRAL": { - # "mistral:mistral-small-latest", + "mistral:mistral-small-latest", # "mistral:mistral-medium-latest", # "mistral:mistral-large-latest", - "mistral:open-mistral-7b", + # "mistral:open-mistral-7b", + # "mistral:open-mixtral-8x7b", }, "Solar": { "solar:solar-1-mini-chat", diff --git a/agent/language_models.py b/agent/language_models.py index 2dcf8f7..52d3eea 100644 --- a/agent/language_models.py +++ b/agent/language_models.py @@ -40,7 +40,6 @@ def get_sync_client(provider: str) -> OpenAI: base_url="https://api.mistral.ai/v1/", api_key=os.getenv("MISTRAL_API_KEY") ) if provider == "solar": - print("solar key", os.getenv("SOLAR_API_KEY")) return OpenAI( base_url="https://api.upstage.ai/v1/solar", api_key=os.getenv("SOLAR_API_KEY") ) diff --git a/eval/game.py b/eval/game.py index bf70f53..2a8907d 100644 --- a/eval/game.py +++ b/eval/game.py @@ -54,7 +54,7 @@ def __init__( solar: bool = False, ): self.nickname = nickname - self.model = model or generate_model(openai=False, mistral=True, solar=False) + self.model = model or generate_model(openai=openai, mistral=mistral, solar=solar) self.openai = False self.mistral = True self.robot = Robot( @@ -80,7 +80,7 @@ def __init__( solar: bool = False, ): self.nickname = nickname - self.model = model or generate_model(openai=False, mistral=False, solar=solar) + self.model = model or generate_model(openai=openai, mistral=mistral, solar=solar) self.robot = Robot( action_space=None, character="Solar", @@ -124,7 +124,8 @@ class Game: render: Optional[bool] = False splash_screen: Optional[bool] = False save_game: Optional[bool] = False - characters: Optional[List[str]] = ["Makoto", "Sean"] + # characters: Optional[List[str]] = ["Makoto", "Sean"] + characters: Optional[List[str]] = ["Ken", "Ken"] outfits: Optional[List[int]] = [1, 3] frame_shape: Optional[List[int]] = [0, 0, 0] seed: Optional[int] = 42 @@ -140,7 +141,8 @@ def __init__( render: bool = False, save_game: bool = False, splash_screen: bool = False, - characters: List[str] = ["Makoto", "Sean"], + characters: List[str] = ["Ken", "Ken"], + # characters: List[str] = ["Makoto", "Sean"], super_arts: List[int] = [3,3], outfits: List[int] = [1, 3], frame_shape: List[int] = [0, 0, 0], @@ -175,17 +177,18 @@ def __init__( self.openai = openai self.mistral = mistral self.solar = solar + print("GAME", openai, mistral, solar, player_1, player_2) self.player_1 = ( player_1 if player_1 - else Player1(nickname="Player 1", openai=self.openai, mistral=self.mistral, solar=self.solar) + else Player1(nickname="Player 1", openai=False, mistral=False, solar=True) # else Player1(nickname="Player 1", model="grok:mixtral-8x7b-32768") ) self.player_2 = ( player_2 if player_2 # else Player2(nickname="Player 2", model="openai:gpt-4-turbo-preview") - else Player2(nickname="Player 2", openai=self.openai, mistral=self.mistral, solar=self.solar) + else Player2(nickname="Player 2", openai=False, mistral=True, solar=False) ) def _init_settings(self) -> EnvironmentSettingsMultiAgent: From 02f071756f52f1a24c58cda0efc3d69be53d17bd Mon Sep 17 00:00:00 2001 From: Ary Date: Tue, 9 Apr 2024 01:30:44 +0800 Subject: [PATCH 4/7] add video --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 52701d8..b26169c 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,12 @@ https://github.com/OpenGenerativeAI/llm-colosseum/assets/19614572/79b58e26-7902- https://github.com/OpenGenerativeAI/llm-colosseum/assets/19614572/5d3d386b-150a-48a5-8f68-7e2954ec18db +1 VS 1: Mistral 7B vs Solar +* mistral-small-latest vs solar-1-mini-chat +* mistral-medium-latest vs solar-1-mini-chat + +* mistral-large-latest vs solar-1-mini-chat ## A new kind of benchmark ? From 5b6ebbfb5152c4a89bdbfcaaf8e2737304e2fa63 Mon Sep 17 00:00:00 2001 From: Ary Date: Tue, 9 Apr 2024 01:31:11 +0800 Subject: [PATCH 5/7] add video --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index b26169c..a597ead 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ https://github.com/OpenGenerativeAI/llm-colosseum/assets/19614572/79b58e26-7902- https://github.com/OpenGenerativeAI/llm-colosseum/assets/19614572/5d3d386b-150a-48a5-8f68-7e2954ec18db -1 VS 1: Mistral 7B vs Solar +### 1 VS 1: Mistral 7B vs Solar * mistral-small-latest vs solar-1-mini-chat * mistral-medium-latest vs solar-1-mini-chat From c67f21e9d00236799a023eb69a1d9c6ecaee4e8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=92=9F=E7=90=A6?= Date: Tue, 9 Apr 2024 01:32:52 +0800 Subject: [PATCH 6/7] Update README.md --- README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a597ead..0082e53 100644 --- a/README.md +++ b/README.md @@ -35,11 +35,12 @@ https://github.com/OpenGenerativeAI/llm-colosseum/assets/19614572/79b58e26-7902- https://github.com/OpenGenerativeAI/llm-colosseum/assets/19614572/5d3d386b-150a-48a5-8f68-7e2954ec18db ### 1 VS 1: Mistral 7B vs Solar -* mistral-small-latest vs solar-1-mini-chat - -* mistral-medium-latest vs solar-1-mini-chat - -* mistral-large-latest vs solar-1-mini-chat +#### mistral-small-latest vs solar-1-mini-chat +https://github.com/Tokkiu/llm-colosseum/assets/13414571/2a7e681d-d022-486c-9250-68fedff4b069 +#### mistral-medium-latest vs solar-1-mini-chat +https://github.com/Tokkiu/llm-colosseum/assets/13414571/d0532e43-11e2-447e-b2b3-6023b9760f11 +#### mistral-large-latest vs solar-1-mini-chat +https://github.com/Tokkiu/llm-colosseum/assets/13414571/4757d562-f800-40ef-8f1a-675b0e23b8ed ## A new kind of benchmark ? From 6f9065f0ea25cbb593560edac65f18ae2442ed52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=AB=98=E7=92=9F=E7=90=A6?= Date: Tue, 9 Apr 2024 01:33:58 +0800 Subject: [PATCH 7/7] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0082e53..2cf0010 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ https://github.com/OpenGenerativeAI/llm-colosseum/assets/19614572/79b58e26-7902- https://github.com/OpenGenerativeAI/llm-colosseum/assets/19614572/5d3d386b-150a-48a5-8f68-7e2954ec18db -### 1 VS 1: Mistral 7B vs Solar +### 1 VS 1: Mistral vs Solar #### mistral-small-latest vs solar-1-mini-chat https://github.com/Tokkiu/llm-colosseum/assets/13414571/2a7e681d-d022-486c-9250-68fedff4b069 #### mistral-medium-latest vs solar-1-mini-chat