Skip to content

Commit

Permalink
update
Browse files Browse the repository at this point in the history
  • Loading branch information
bowenliang123 committed Sep 24, 2024
1 parent 45ea694 commit 9be4f1c
Show file tree
Hide file tree
Showing 30 changed files with 67 additions and 54 deletions.
2 changes: 1 addition & 1 deletion api/core/agent/cot_agent_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -369,7 +369,7 @@ def _format_assistant_message(self, agent_scratchpad: list[AgentScratchpadUnit])
return message

def _organize_historic_prompt_messages(
self, current_session_messages: list[PromptMessage] | None = None
self, current_session_messages: Optional[list[PromptMessage]] = None
) -> list[PromptMessage]:
"""
organize historic prompt messages
Expand Down
3 changes: 2 additions & 1 deletion api/core/agent/cot_chat_agent_runner.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
from typing import Optional

from core.agent.cot_agent_runner import CotAgentRunner
from core.model_runtime.entities.message_entities import (
Expand Down Expand Up @@ -27,7 +28,7 @@ def _organize_system_prompt(self) -> SystemPromptMessage:

return SystemPromptMessage(content=system_prompt)

def _organize_user_query(self, query, prompt_messages: list[PromptMessage] | None = None) -> list[PromptMessage]:
def _organize_user_query(self, query, prompt_messages: Optional[list[PromptMessage]] = None) -> list[PromptMessage]:
"""
Organize user query
"""
Expand Down
3 changes: 2 additions & 1 deletion api/core/agent/cot_completion_agent_runner.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import json
from typing import Optional

from core.agent.cot_agent_runner import CotAgentRunner
from core.model_runtime.entities.message_entities import AssistantPromptMessage, PromptMessage, UserPromptMessage
Expand All @@ -21,7 +22,7 @@ def _organize_instruction_prompt(self) -> str:

return system_prompt

def _organize_historic_prompt(self, current_session_messages: list[PromptMessage] | None = None) -> str:
def _organize_historic_prompt(self, current_session_messages: Optional[list[PromptMessage]] = None) -> str:
"""
Organize historic prompt
"""
Expand Down
6 changes: 3 additions & 3 deletions api/core/agent/fc_agent_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@
import logging
from collections.abc import Generator
from copy import deepcopy
from typing import Any, Union
from typing import Any, Optional, Union

from core.agent.base_agent_runner import BaseAgentRunner
from core.app.apps.base_app_queue_manager import PublishFrom
Expand Down Expand Up @@ -370,7 +370,7 @@ def extract_blocking_tool_calls(self, llm_result: LLMResult) -> Union[None, list
return tool_calls

def _init_system_message(
self, prompt_template: str, prompt_messages: list[PromptMessage] | None = None
self, prompt_template: str, prompt_messages: Optional[list[PromptMessage]] = None
) -> list[PromptMessage]:
"""
Initialize system message
Expand All @@ -385,7 +385,7 @@ def _init_system_message(

return prompt_messages

def _organize_user_query(self, query, prompt_messages: list[PromptMessage] | None = None) -> list[PromptMessage]:
def _organize_user_query(self, query, prompt_messages: Optional[list[PromptMessage]] = None) -> list[PromptMessage]:
"""
Organize user query
"""
Expand Down
4 changes: 2 additions & 2 deletions api/core/indexing_runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -211,9 +211,9 @@ def indexing_estimate(
tenant_id: str,
extract_settings: list[ExtractSetting],
tmp_processing_rule: dict,
doc_form: str | None = None,
doc_form: Optional[str] = None,
doc_language: str = "English",
dataset_id: str | None = None,
dataset_id: Optional[str] = None,
indexing_technique: str = "economy",
) -> dict:
"""
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -511,7 +511,7 @@ def _num_tokens_from_messages(
model: str,
messages: list[PromptMessage],
tools: Optional[list[PromptMessageTool]] = None,
credentials: dict | None = None,
credentials: Optional[dict] = None,
) -> int:
"""
Approximate num tokens with GPT2 tokenizer.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -688,7 +688,7 @@ def _num_tokens_from_messages(
model: str,
messages: list[PromptMessage],
tools: Optional[list[PromptMessageTool]] = None,
credentials: dict | None = None,
credentials: Optional[dict] = None,
) -> int:
"""
Approximate num tokens with GPT2 tokenizer.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -77,7 +77,7 @@ def validate_credentials(self, model: str, credentials: dict) -> None:
"""
pass

def _detect_lang_code(self, content: str, map_dict: dict | None = None):
def _detect_lang_code(self, content: str, map_dict: Optional[dict] = None):
map_dict = {"zh": "<|zh|>", "en": "<|en|>", "ja": "<|jp|>", "zh-TW": "<|yue|>", "ko": "<|ko|>"}

response = self.comprehend_client.detect_dominant_language(Text=content)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,7 @@ def conversation(
def query_support(
self,
*,
assistant_id_list: list[str] | None = None,
assistant_id_list: Optional[list[str]] = None,
request_id: Optional[str] = None,
user_id: Optional[str] = None,
extra_headers: Headers | None = None,
Expand Down
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
from __future__ import annotations

from collections.abc import Mapping
from typing import TYPE_CHECKING, Literal, cast
from typing import TYPE_CHECKING, Literal, Optional, cast

import httpx

Expand Down Expand Up @@ -35,10 +35,10 @@ def create(
self,
*,
file: FileTypes = None,
upload_detail: list[UploadDetail] | None = None,
upload_detail: Optional[list[UploadDetail]] = None,
purpose: Literal["fine-tune", "retrieval", "batch"],
knowledge_id: str | None = None,
sentence_size: int | None = None,
knowledge_id: Optional[str] = None,
sentence_size: Optional[int] = None,
extra_headers: Headers | None = None,
extra_body: Body | None = None,
timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,10 +36,10 @@ def create(
*,
file: FileTypes = None,
custom_separator: Optional[list[str]] = None,
upload_detail: list[UploadDetail] | None = None,
upload_detail: Optional[list[UploadDetail]] = None,
purpose: Literal["retrieval"],
knowledge_id: str | None = None,
sentence_size: int | None = None,
knowledge_id: Optional[str] = None,
sentence_size: Optional[int] = None,
extra_headers: Headers | None = None,
extra_body: Body | None = None,
timeout: float | httpx.Timeout | None | NotGiven = NOT_GIVEN,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ def generations(
self,
model: str,
*,
prompt: str | None = None,
image_url: str | None = None,
prompt: Optional[str] = None,
image_url: Optional[str] = None,
sensitive_word_check: Optional[SensitiveWordCheckRequest] | NotGiven = NOT_GIVEN,
request_id: Optional[str] = None,
user_id: Optional[str] = None,
Expand Down
2 changes: 1 addition & 1 deletion api/core/rag/datasource/vdb/relyt/relyt_vector.py
Original file line number Diff line number Diff line change
Expand Up @@ -162,7 +162,7 @@ def get_ids_by_metadata_field(self, key: str, value: str):
else:
return None

def delete_by_uuids(self, ids: list[str] | None = None):
def delete_by_uuids(self, ids: Optional[list[str]] = None):
"""Delete by vector IDs.
Args:
Expand Down
6 changes: 3 additions & 3 deletions api/core/rag/datasource/vdb/vector_factory.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from abc import ABC, abstractmethod
from typing import Any
from typing import Any, Optional

from configs import dify_config
from core.embedding.cached_embedding import CacheEmbedding
Expand All @@ -25,7 +25,7 @@ def gen_index_struct_dict(vector_type: VectorType, collection_name: str) -> dict


class Vector:
def __init__(self, dataset: Dataset, attributes: list | None = None):
def __init__(self, dataset: Dataset, attributes: Optional[list] = None):
if attributes is None:
attributes = ["doc_id", "dataset_id", "document_id", "doc_hash"]
self._dataset = dataset
Expand Down Expand Up @@ -106,7 +106,7 @@ def get_vector_factory(vector_type: str) -> type[AbstractVectorFactory]:
case _:
raise ValueError(f"Vector store {vector_type} is not supported.")

def create(self, texts: list | None = None, **kwargs):
def create(self, texts: Optional[list] = None, **kwargs):
if texts:
embeddings = self._embeddings.embed_documents([document.page_content for document in texts])
self._vector_processor.create(texts=texts, embeddings=embeddings, **kwargs)
Expand Down
4 changes: 2 additions & 2 deletions api/core/rag/extractor/extract_processor.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
import re
import tempfile
from pathlib import Path
from typing import Union
from typing import Optional, Union
from urllib.parse import unquote

from configs import dify_config
Expand Down Expand Up @@ -83,7 +83,7 @@ def load_from_url(cls, url: str, return_text: bool = False) -> Union[list[Docume

@classmethod
def extract(
cls, extract_setting: ExtractSetting, is_automatic: bool = False, file_path: str | None = None
cls, extract_setting: ExtractSetting, is_automatic: bool = False, file_path: Optional[str] = None
) -> list[Document]:
if extract_setting.datasource_type == DatasourceType.FILE.value:
with tempfile.TemporaryDirectory() as temp_dir:
Expand Down
2 changes: 1 addition & 1 deletion api/core/tools/entities/tool_entities.py
Original file line number Diff line number Diff line change
Expand Up @@ -341,7 +341,7 @@ def set_text(self, tool_name: str, name: str, value: str) -> None:

self.pool.append(variable)

def set_file(self, tool_name: str, value: str, name: str | None = None) -> None:
def set_file(self, tool_name: str, value: str, name: Optional[str] = None) -> None:
"""
set an image variable
Expand Down
4 changes: 2 additions & 2 deletions api/core/tools/provider/builtin/aws/tools/sagemaker_tts.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
import json
from enum import Enum
from typing import Any, Union
from typing import Any, Optional, Union

import boto3

Expand All @@ -21,7 +21,7 @@ class SageMakerTTSTool(BuiltinTool):
s3_client: Any = None
comprehend_client: Any = None

def _detect_lang_code(self, content: str, map_dict: dict | None = None):
def _detect_lang_code(self, content: str, map_dict: Optional[dict] = None):
map_dict = {"zh": "<|zh|>", "en": "<|en|>", "ja": "<|jp|>", "zh-TW": "<|yue|>", "ko": "<|ko|>"}

response = self.comprehend_client.detect_dominant_language(Text=content)
Expand Down
4 changes: 3 additions & 1 deletion api/core/tools/tool/builtin_tool.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from typing import Optional

from core.model_runtime.entities.llm_entities import LLMResult
from core.model_runtime.entities.message_entities import PromptMessage, SystemPromptMessage, UserPromptMessage
from core.tools.entities.tool_entities import ToolProviderType
Expand Down Expand Up @@ -124,7 +126,7 @@ def summarize(content: str) -> str:

return result

def get_url(self, url: str, user_agent: str | None = None) -> str:
def get_url(self, url: str, user_agent: Optional[str] = None) -> str:
"""
get url
"""
Expand Down
4 changes: 2 additions & 2 deletions api/core/tools/tool_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
from collections.abc import Generator
from os import listdir, path
from threading import Lock
from typing import Any, Union
from typing import Any, Optional, Union

from configs import dify_config
from core.agent.entities import AgentToolEntity
Expand Down Expand Up @@ -72,7 +72,7 @@ def get_builtin_tool(cls, provider: str, tool_name: str) -> BuiltinTool:

@classmethod
def get_tool(
cls, provider_type: str, provider_id: str, tool_name: str, tenant_id: str | None = None
cls, provider_type: str, provider_id: str, tool_name: str, tenant_id: Optional[str] = None
) -> Union[BuiltinTool, ApiTool]:
"""
get the tool
Expand Down
6 changes: 4 additions & 2 deletions api/core/tools/utils/feishu_api_utils.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
from typing import Optional

import httpx

from extensions.ext_redis import redis_client
Expand All @@ -22,8 +24,8 @@ def _send_request(
url: str,
method: str = "post",
require_token: bool = True,
payload: dict | None = None,
params: dict | None = None,
payload: Optional[dict] = None,
params: Optional[dict] = None,
):
headers = {
"Content-Type": "application/json",
Expand Down
11 changes: 6 additions & 5 deletions api/core/tools/utils/parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
from json import dumps as json_dumps
from json import loads as json_loads
from json.decoder import JSONDecodeError
from typing import Optional

from requests import get
from yaml import YAMLError, safe_load
Expand All @@ -16,7 +17,7 @@
class ApiBasedToolSchemaParser:
@staticmethod
def parse_openapi_to_tool_bundle(
openapi: dict, extra_info: dict | None = None, warning: dict | None = None
openapi: dict, extra_info: Optional[dict], warning: Optional[dict]
) -> list[ApiToolBundle]:
warning = warning if warning is not None else {}
extra_info = extra_info if extra_info is not None else {}
Expand Down Expand Up @@ -174,7 +175,7 @@ def _get_tool_parameter_type(parameter: dict) -> ToolParameter.ToolParameterType

@staticmethod
def parse_openapi_yaml_to_tool_bundle(
yaml: str, extra_info: dict | None = None, warning: dict | None = None
yaml: str, extra_info: Optional[dict], warning: Optional[dict]
) -> list[ApiToolBundle]:
"""
parse openapi yaml to tool bundle
Expand All @@ -191,7 +192,7 @@ def parse_openapi_yaml_to_tool_bundle(
return ApiBasedToolSchemaParser.parse_openapi_to_tool_bundle(openapi, extra_info=extra_info, warning=warning)

@staticmethod
def parse_swagger_to_openapi(swagger: dict, extra_info: dict | None = None, warning: dict | None = None) -> dict:
def parse_swagger_to_openapi(swagger: dict, extra_info: Optional[dict], warning: Optional[dict]) -> dict:
"""
parse swagger to openapi
Expand Down Expand Up @@ -253,7 +254,7 @@ def parse_swagger_to_openapi(swagger: dict, extra_info: dict | None = None, warn

@staticmethod
def parse_openai_plugin_json_to_tool_bundle(
json: str, extra_info: dict | None = None, warning: dict | None = None
json: str, extra_info: Optional[dict], warning: Optional[dict]
) -> list[ApiToolBundle]:
"""
parse openapi plugin yaml to tool bundle
Expand Down Expand Up @@ -287,7 +288,7 @@ def parse_openai_plugin_json_to_tool_bundle(

@staticmethod
def auto_parse_to_tool_bundle(
content: str, extra_info: dict | None = None, warning: dict | None = None
content: str, extra_info: Optional[dict], warning: Optional[dict]
) -> tuple[list[ApiToolBundle], str]:
"""
auto parse to tool bundle
Expand Down
2 changes: 1 addition & 1 deletion api/core/tools/utils/web_reader_tool.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ def page_result(text: str, cursor: int, max_length: int) -> str:
return text[cursor : cursor + max_length]


def get_url(url: str, user_agent: str | None = None) -> str:
def get_url(url: str, user_agent: Optional[str] = None) -> str:
"""Fetch URL and return the contents as a string."""
headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko)"
Expand Down
2 changes: 1 addition & 1 deletion api/libs/helper.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,7 @@ def generate() -> Generator:

class TokenManager:
@classmethod
def generate_token(cls, account: Account, token_type: str, additional_data: dict | None = None) -> str:
def generate_token(cls, account: Account, token_type: str, additional_data: Optional[dict] = None) -> str:
old_token = cls._get_current_token_for_account(account.id, token_type)
if old_token:
if isinstance(old_token, bytes):
Expand Down
2 changes: 1 addition & 1 deletion api/services/account_service.py
Original file line number Diff line number Diff line change
Expand Up @@ -321,7 +321,7 @@ def get_current_tenant_by_account(account: Account):
return tenant

@staticmethod
def switch_tenant(account: Account, tenant_id: int | None = None) -> None:
def switch_tenant(account: Account, tenant_id: Optional[int] = None) -> None:
"""Switch the current workspace for the account"""

# Ensure tenant_id is provided
Expand Down
5 changes: 4 additions & 1 deletion api/services/errors/base.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,6 @@
from typing import Optional


class BaseServiceError(Exception):
def __init__(self, description: str | None = None):
def __init__(self, description: Optional[str] = None):
self.description = description
3 changes: 2 additions & 1 deletion api/services/tag_service.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import uuid
from typing import Optional

from flask_login import current_user
from sqlalchemy import func
Expand All @@ -11,7 +12,7 @@

class TagService:
@staticmethod
def get_tags(tag_type: str, current_tenant_id: str, keyword: str | None = None) -> list:
def get_tags(tag_type: str, current_tenant_id: str, keyword: Optional[str] = None) -> list:
query = (
db.session.query(Tag.id, Tag.type, Tag.name, func.count(TagBinding.id).label("binding_count"))
.outerjoin(TagBinding, Tag.id == TagBinding.tag_id)
Expand Down
Loading

0 comments on commit 9be4f1c

Please sign in to comment.