Merge branch 'main' into access-to-all-columns

.all-contributorsrc

@@ -417,6 +417,16 @@
       "contributions": [
         "doc"
       ]
+    },
+    {
+      "login": "LucasTrg",
+      "name": "LucasTrg",
+      "avatar_url": "https://avatars.githubusercontent.com/u/47852577?v=4",
+      "profile": "https://github.com/LucasTrg",
+      "contributions": [
+        "code",
+        "bug"
+      ]
     }
   ],
   "contributorsPerLine": 7,

.github/actions/check-app-accessibility/action.yml

@@ -0,0 +1,40 @@
+name: 'Check App Accessibility'
+description: 'Parse logs and check if the app is running'
+inputs:
+  log-file:
+    description: 'The path to the log file containing the serve output'
+    required: true
+runs:
+  using: "composite"
+  steps:
+    - name: Parse logs and check if app is running
+      id: check-app
+      shell: bash
+      run: |
+
+        echo "Serve output:"
+        cat ${{ inputs.log-file }}
+        APP_URL=$(grep -oP 'You can access it here: \K.+' ${{ inputs.log-file }} | head -n 1)
+        echo "Extracted app URL: $APP_URL"
+        if [ -z "$APP_URL" ]; then
+            echo "Error: Failed to extract app URL from logs."
+            exit 1
+        fi
+        APP_URL="${APP_URL%/}/openapi.json"
+
+        echo "Checking if $APP_URL is accessible"
+        retries=6
+        for i in $(seq 1 $retries); do
+            status_code=$(curl --max-time 60 --write-out %{http_code} --silent --output /dev/null --verbose $APP_URL)
+            echo "Attempt $i: Status code: $status_code"
+            if [ "$status_code" -eq 200 ]; then
+                echo "$APP_URL is accessible"
+                exit 0
+            else
+                echo "Attempt $i failed: $APP_URL is not accessible"
+                sleep 10 # Wait before retrying
+            fi
+        done
+
+        echo "Error: $APP_URL is not accessible after $retries attempts"
+        exit 1

.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "pip"
+    directory: "/agenta-cli" # The directory where your pyproject.toml file is located
+    schedule:
+      interval: "daily" # Check for updates daily

.github/workflows/cli-commands-tests.yml

@@ -43,10 +43,16 @@ jobs:
             - name: Run agenta variant serve
               run: |
                   cd examples/baby_name_generator
-                  agenta variant serve --file_name app.py
+                  agenta variant serve --file_name app.py 2>&1 | tee ../../serve_output.log
               shell: bash
               continue-on-error: false
 
+            - name: Check if app is running
+              uses: ./.github/actions/check-app-accessibility
+              with:
+                log-file: serve_output.log
+              continue-on-error: false
+
             - name: Run agenta variant serve with overwrite
               run: |
                   cd examples/baby_name_generator
@@ -90,10 +96,16 @@ jobs:
             - name: Run agenta variant serve
               run: |
                   cd examples/baby_name_generator
-                  agenta variant serve --file_name app.py
+                  agenta variant serve --file_name app.py 2>&1 | tee ../../serve_output.log
               shell: bash
               continue-on-error: false
 
+            - name: Check if app is running
+              uses: ./.github/actions/check-app-accessibility
+              with:
+                log-file: serve_output.log
+              continue-on-error: false
+
             - name: Run agenta variant serve with overwrite
               run: |
                   cd examples/baby_name_generator

.gitignore

@@ -64,3 +64,4 @@ agenta-backend/agenta_backend/migrations/migrations/v0_11_0_to_v0_12_0/202402030
 agenta-backend/agenta_backend/migrations/migrations/v0_11_0_to_v0_12_0/20240201102156_update_org_invitation.py
 agenta-backend/agenta_backend/migrations/migrations/v0_11_0_to_v0_12_0/20240203035520_add_workspace_field_to_db_models.py
 docker-compose.oss.prod.yml
+agenta-backend/agenta_backend/services/benchmark.ipynb

README.md

@@ -169,7 +169,7 @@ Check out our [Contributing Guide](https://docs.agenta.ai/contributing/getting-s
 ## Contributors ✨
 
 <!-- ALL-CONTRIBUTORS-BADGE:START - Do not remove or modify this section -->
-[![All Contributors](https://img.shields.io/badge/all_contributors-44-orange.svg?style=flat-square)](#contributors-)
+[![All Contributors](https://img.shields.io/badge/all_contributors-45-orange.svg?style=flat-square)](#contributors-)
 <!-- ALL-CONTRIBUTORS-BADGE:END -->
 
 Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/docs/en/emoji-key)):
@@ -236,6 +236,7 @@ Thanks goes to these wonderful people ([emoji key](https://allcontributors.org/d
     <tr>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/vishalvanpariya"><img src="https://avatars.githubusercontent.com/u/27823328?v=4?s=100" width="100px;" alt="Vishal Vanpariya"/><br /><sub><b>Vishal Vanpariya</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=vishalvanpariya" title="Code">💻</a></td>
       <td align="center" valign="top" width="14.28%"><a href="https://github.com/youcefs21"><img src="https://avatars.githubusercontent.com/u/34604972?v=4?s=100" width="100px;" alt="Youcef Boumar"/><br /><sub><b>Youcef Boumar</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=youcefs21" title="Documentation">📖</a></td>
+      <td align="center" valign="top" width="14.28%"><a href="https://github.com/LucasTrg"><img src="https://avatars.githubusercontent.com/u/47852577?v=4?s=100" width="100px;" alt="LucasTrg"/><br /><sub><b>LucasTrg</b></sub></a><br /><a href="https://github.com/Agenta-AI/agenta/commits?author=LucasTrg" title="Code">💻</a> <a href="https://github.com/Agenta-AI/agenta/issues?q=author%3ALucasTrg" title="Bug reports">🐛</a></td>
     </tr>
   </tbody>
 </table>

agenta-backend/agenta_backend/models/api/evaluation_model.py

@@ -252,6 +252,7 @@ class LMProvidersEnum(str, Enum):
     alephalpha = "ALEPHALPHA_API_KEY"
     openrouter = "OPENROUTER_API_KEY"
     groq = "GROQ_API_KEY"
+    gemini = "GEMINI_API_KEY"
 
 
 class NewEvaluation(BaseModel):

agenta-backend/agenta_backend/models/converters.py

@@ -363,6 +363,7 @@ async def environment_db_to_output(
         deployed_variant_name = None
         revision = None
 
+    await environment_db.fetch_link(AppEnvironmentDB.deployed_app_variant_revision)
     environment_output = EnvironmentOutput(
         name=environment_db.name,
         app_id=str(environment_db.app.id),

agenta-backend/agenta_backend/routers/app_router.py

@@ -578,6 +578,7 @@ async def create_app_and_variant_from_template(
                 "ALEPHALPHA_API_KEY",
                 "OPENROUTER_API_KEY",
                 "GROQ_API_KEY",
+                "GEMINI_API_KEY",
             ]
             missing_keys = [
                 key for key in supported_llm_prodviders_keys if not os.environ.get(key)

agenta-backend/agenta_backend/routers/configs_router.py

@@ -93,7 +93,7 @@ async def get_config(
     try:
         base_db = await db_manager.fetch_base_by_id(base_id)
 
-        # detemine whether the user has access to the base
+        # determine whether the user has access to the base
         if isCloudEE():
             has_permission = await check_action_access(
                 user_uid=request.state.user_id,
@@ -111,15 +111,16 @@ async def get_config(
         # in case environment_name is provided, find the variant deployed
         if environment_name:
             app_environments = await db_manager.list_environments(
-                app_id=str(base_db.app.id)
+                app_id=str(base_db.app.ref.id)
+            )
+            found_variant_revision = next(
+                (
+                    app_environment.deployed_app_variant_revision
+                    for app_environment in app_environments
+                    if app_environment.name == environment_name
+                ),
+                None,
             )
-            found_variant = None
-            for app_environment in app_environments:
-                if app_environment.name == environment_name:
-                    found_variant_revision = (
-                        app_environment.deployed_app_variant_revision
-                    )
-                    break
             if not found_variant_revision:
                 raise HTTPException(
                     status_code=400,
@@ -134,11 +135,14 @@ async def get_config(
             config = found_variant_revision.config
         elif config_name:
             variants_db = await db_manager.list_variants_for_base(base_db)
-            found_variant = None
-            for variant_db in variants_db:
-                if variant_db.config_name == config_name:
-                    found_variant = variant_db
-                    break
+            found_variant = next(
+                (
+                    variant_db
+                    for variant_db in variants_db
+                    if variant_db.config_name == config_name
+                ),
+                None,
+            )
             if not found_variant:
                 raise HTTPException(
                     status_code=400,

agenta-backend/agenta_backend/services/db_manager.py

@@ -258,14 +258,15 @@ async def fetch_base_by_id(base_id: str) -> Optional[VariantBaseDB]:
     Returns:
         VariantBaseDB: The fetched base, or None if no base was found.
     """
+
     if base_id is None:
         raise Exception("No base_id provided")
-    base = await VariantBaseDB.find_one(
-        VariantBaseDB.id == ObjectId(base_id), fetch_links=True
-    )
-    if base is None:
+
+    base = await VariantBaseDB.find_one(VariantBaseDB.id == ObjectId(base_id))
+    if not base:
         logger.error("Base not found")
-        return False
+        return None
+
     return base
 
 
@@ -652,9 +653,7 @@ async def list_variants_for_base(base: VariantBaseDB) -> List[AppVariantDB]:
     """
     assert base is not None, "base cannot be None"
     app_variants_db = (
-        await AppVariantDB.find(
-            AppVariantDB.base.id == ObjectId(base.id), fetch_links=True
-        )
+        await AppVariantDB.find(AppVariantDB.base.id == ObjectId(base.id))
         .sort("variant_name")
         .to_list()
     )
@@ -865,6 +864,10 @@ async def add_variant_from_base_and_config(
         config_name=new_config_name,
         parameters=parameters,
     )
+
+    # Prefetch image in base_db
+    await base_db.fetch_link(VariantBaseDB.image)
+
     db_app_variant = AppVariantDB(
         app=previous_app_variant_db.app,
         variant_name=new_variant_name,

agenta-backend/agenta_backend/services/llm_apps_service.py

@@ -172,6 +172,7 @@ async def run_with_retry(
             last_exception = e
             logger.info(f"Error processing datapoint: {input_data}. {str(e)}")
             logger.info("".join(traceback.format_exception_only(type(e), e)))
+            retries += 1
             common.capture_exception_in_sentry(e)
 
     # If max retries is reached or an exception that isn't in the second block,
@@ -186,7 +187,7 @@ async def run_with_retry(
         result=Result(
             type="error",
             value=None,
-            error=Error(message=exception_message, stacktrace=last_exception),
+            error=Error(message=exception_message, stacktrace=str(last_exception)),
         )
     )
 

agenta-backend/agenta_backend/services/security/sandbox.py

@@ -91,7 +91,7 @@ def execute_code_safely(
 
     # Call the evaluation function, extract the result if it exists
     # and is a float between 0 and 1
-    result = environment["evaluate"](app_params, inputs, correct_answer, output)
+    result = environment["evaluate"](app_params, inputs, output, correct_answer)
     if isinstance(result, float) and 0 <= result <= 1:
         return result
     return None