diff --git a/agenta-backend/agenta_backend/services/evaluators_service.py b/agenta-backend/agenta_backend/services/evaluators_service.py index 5bd13383d4..f3c0d25e54 100644 --- a/agenta-backend/agenta_backend/services/evaluators_service.py +++ b/agenta-backend/agenta_backend/services/evaluators_service.py @@ -275,8 +275,7 @@ def auto_ai_critique( model="gpt-3.5-turbo", messages=messages, temperature=0.8 ) - evaluation_output = response.choices[0].message["content"].strip() - + evaluation_output = response.choices[0].message.content.strip() return Result(type="text", value=evaluation_output) except Exception as e: # pylint: disable=broad-except return Result( diff --git a/agenta-backend/pyproject.toml b/agenta-backend/pyproject.toml index c1bd18e228..245e40d677 100644 --- a/agenta-backend/pyproject.toml +++ b/agenta-backend/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "agenta_backend" -version = "0.17.1" +version = "0.17.2" description = "" authors = ["Mahmoud Mabrouk "] readme = "README.md" diff --git a/agenta-cli/agenta/client/backend/types/llm_tokens.py b/agenta-cli/agenta/client/backend/types/llm_tokens.py index 0c5a71755b..7e9fd8358b 100644 --- a/agenta-cli/agenta/client/backend/types/llm_tokens.py +++ b/agenta-cli/agenta/client/backend/types/llm_tokens.py @@ -33,6 +33,6 @@ def dict(self, **kwargs: typing.Any) -> typing.Dict[str, typing.Any]: return super().dict(**kwargs_with_defaults) class Config: - frozen = True + frozen = False smart_union = True json_encoders = {dt.datetime: serialize_datetime} diff --git a/agenta-cli/agenta/sdk/tracing/llm_tracing.py b/agenta-cli/agenta/sdk/tracing/llm_tracing.py index d1309c3015..23a806569a 100644 --- a/agenta-cli/agenta/sdk/tracing/llm_tracing.py +++ b/agenta-cli/agenta/sdk/tracing/llm_tracing.py @@ -7,7 +7,11 @@ from agenta.sdk.tracing.tasks_manager import TaskQueue from agenta.client.backend.client import AsyncAgentaApi from agenta.client.backend.client import AsyncObservabilityClient -from agenta.client.backend.types.create_span import CreateSpan, SpanStatusCode +from agenta.client.backend.types.create_span import ( + CreateSpan, + LlmTokens, + SpanStatusCode, +) from bson.objectid import ObjectId @@ -173,6 +177,24 @@ def start_span( def update_span_status(self, span: CreateSpan, value: str): span.status = value + def _update_span_cost(self, span: CreateSpan, cost: Optional[float]): + if cost is not None and isinstance(cost, float): + if span.cost is None: + span.cost = cost + else: + span.cost += cost + + def _update_span_tokens(self, span: CreateSpan, tokens: Optional[dict]): + if isinstance(tokens, LlmTokens): + tokens = tokens.dict() + if tokens is not None and isinstance(tokens, dict): + if span.tokens is None: + span.tokens = LlmTokens(**tokens) + else: + span.tokens.prompt_tokens += tokens["prompt_tokens"] + span.tokens.completion_tokens += tokens["completion_tokens"] + span.tokens.total_tokens += tokens["total_tokens"] + def end_span(self, outputs: Dict[str, Any]): """ Ends the active span, if it is a parent span, ends the trace too. @@ -196,8 +218,12 @@ def end_span(self, outputs: Dict[str, Any]): self.active_span.end_time = datetime.now(timezone.utc) self.active_span.outputs = [outputs.get("message", "")] - self.active_span.cost = outputs.get("cost", None) - self.active_span.tokens = outputs.get("usage", None) + if self.active_span.spankind in [ + "LLM", + "RETRIEVER", + ]: # TODO: Remove this whole part. Setting the cost should be done through set_span_attribute + self._update_span_cost(self.active_span, outputs.get("cost", None)) + self._update_span_tokens(self.active_span, outputs.get("usage", None)) # Push span to list of recorded spans self.pending_spans.append(self.active_span) @@ -213,7 +239,10 @@ def end_span(self, outputs: Dict[str, Any]): self.end_trace(parent_span=self.active_span) else: - self.active_span = self.span_dict.get(active_span_parent_id) + parent_span = self.span_dict[active_span_parent_id] + self._update_span_cost(parent_span, self.active_span.cost) + self._update_span_tokens(parent_span, self.active_span.tokens) + self.active_span = parent_span def record_exception_and_end_trace(self, span_parent_id: str): """ diff --git a/agenta-cli/pyproject.toml b/agenta-cli/pyproject.toml index 3e3f8ddf86..ef66c050f5 100644 --- a/agenta-cli/pyproject.toml +++ b/agenta-cli/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "agenta" -version = "0.17.1" +version = "0.17.2" description = "The SDK for agenta is an open-source LLMOps platform." readme = "README.md" authors = ["Mahmoud Mabrouk "] diff --git a/agenta-web/package-lock.json b/agenta-web/package-lock.json index cf0a5728af..1720598967 100644 --- a/agenta-web/package-lock.json +++ b/agenta-web/package-lock.json @@ -1,12 +1,12 @@ { "name": "agenta", - "version": "0.17.1", + "version": "0.17.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "agenta", - "version": "0.17.1", + "version": "0.17.2", "dependencies": { "@ant-design/colors": "^7.0.0", "@ant-design/icons": "^5.0.1", diff --git a/agenta-web/package.json b/agenta-web/package.json index 1b8b4bc69d..1d3acde5e1 100644 --- a/agenta-web/package.json +++ b/agenta-web/package.json @@ -1,6 +1,6 @@ { "name": "agenta", - "version": "0.17.1", + "version": "0.17.2", "private": true, "engines": { "node": ">=18" diff --git a/docker-compose.test.yml b/docker-compose.test.yml index 61b1a974a2..07cf5ce4e6 100644 --- a/docker-compose.test.yml +++ b/docker-compose.test.yml @@ -25,7 +25,7 @@ services: - DATABASE_MODE=test - FEATURE_FLAG=oss - OPENAI_API_KEY=${OPENAI_API_KEY} - - AGENTA_TEMPLATE_REPO=agentaai/stage_templates + - AGENTA_TEMPLATE_REPO=agentaai/templates_v2 - POSTHOG_API_KEY=phc_hmVSxIjTW1REBHXgj2aw4HW9X6CXb6FzerBgP9XenC7 - TEMPLATES_BASE_URL=https://llm-app-json.s3.eu-central-1.amazonaws.com - REGISTRY_REPO_NAME=agentaai diff --git a/docs/images/misc/team_management_dark.png b/docs/images/misc/team_management_dark.png new file mode 100644 index 0000000000..6f7659b184 Binary files /dev/null and b/docs/images/misc/team_management_dark.png differ diff --git a/docs/images/misc/team_management_light.png b/docs/images/misc/team_management_light.png new file mode 100644 index 0000000000..081cded50d Binary files /dev/null and b/docs/images/misc/team_management_light.png differ diff --git a/docs/images/misc/workspace_tab_dark.png b/docs/images/misc/workspace_tab_dark.png new file mode 100644 index 0000000000..18ad95ff2a Binary files /dev/null and b/docs/images/misc/workspace_tab_dark.png differ diff --git a/docs/images/misc/workspace_tab_light.png b/docs/images/misc/workspace_tab_light.png new file mode 100644 index 0000000000..745474e74a Binary files /dev/null and b/docs/images/misc/workspace_tab_light.png differ diff --git a/docs/misc/team_management.mdx b/docs/misc/team_management.mdx index 9e21617229..e25a609aa2 100644 --- a/docs/misc/team_management.mdx +++ b/docs/misc/team_management.mdx @@ -2,19 +2,29 @@ title: 'Team Management' --- - Role-based access control and audit trails are currently under development. +## Inviting users +To add new users to your workspace, start by clicking on **Settings** in the lower left corner of the screen. Then, navigate to the **Workspace** tab and on the far right, you will find the **Invite** button. Clicking this button will open a modal window where you can enter the email address(es) of the user(s) you wish to invite and select their role from a dropdown menu. Once you have entered the necessary information, click the **"Invite"** button to send the invitation. The user(s) will then receive an email invitation to join your workspace. -## Inviting users + + + +### User Roles and Their Rights +- **Owner**: The Owner has full management capabilities within the workspace. This includes adding and removing members, as well as managing all settings and content. Essentially, the Owner has unrestricted access to all features and controls. + +- **Workspace Admin**: A Workspace Admin can manage most aspects of the workspace, including settings and member management. However, they do not have the authority to delete the workspace itself. + +- **Editor**: Editors are responsible for editing workspace content. They do not have the ability to manage members or assign roles, focusing solely on content management. + +- **Evaluator**: Evaluators are tasked with evaluating models and providing feedback. They do not have editing privileges or member management rights, ensuring their role is concentrated on assessment and feedback. -To add new users to your workspace, click on settings in the lower left corner of the screen. Then select the "Workspace" tab. You can invite new users by entering their email addresses. +- **Deployment Manager**: Deployment Managers handle the management of model deployments within the workspace. Similar to Evaluators, they do not have content editing or member management rights, allowing them to specialize in deployment tasks. - - +- **Viewer**: Viewers can access and view all workspace content, but they are restricted from making any changes or managing members and roles. This role is typically for those who need to monitor progress without intervening in the workspace operations. ## Switching workspaces If you're a member of multiple workspaces, you can switch between them by clicking the workspace name in the lower left corner of the screen. Remember, all applications in a workspace are shared among all its members. - - \ No newline at end of file + + \ No newline at end of file