From 260e9bcccf377a8a0ca561c10377748e5cc92ebb Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Thu, 21 Dec 2023 18:08:25 +0800 Subject: [PATCH 01/63] Fix makefiles --- erniebot-agent/Makefile | 5 ++++- erniebot/Makefile | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/erniebot-agent/Makefile b/erniebot-agent/Makefile index 0b198b07a..1f527f474 100644 --- a/erniebot-agent/Makefile +++ b/erniebot-agent/Makefile @@ -1,6 +1,9 @@ -.DEFAULT_GOAL = format lint type_check +.DEFAULT_GOAL = dev files_to_format_and_lint = src examples tests +.PHONY: dev +dev: format lint type_check + .PHONY: format format: python -m black $(files_to_format_and_lint) diff --git a/erniebot/Makefile b/erniebot/Makefile index f547eff63..cd2c1f7ea 100644 --- a/erniebot/Makefile +++ b/erniebot/Makefile @@ -1,6 +1,9 @@ -.DEFAULT_GOAL = format lint type_check +.DEFAULT_GOAL = dev files_to_format_and_lint = src examples tests +.PHONY: dev +dev: format lint type_check + .PHONY: format format: python -m black $(files_to_format_and_lint) From 8b23664296be7a9ed91d7d2282aa7d8ba2522674 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Thu, 21 Dec 2023 18:09:06 +0800 Subject: [PATCH 02/63] Fix bugs --- erniebot/src/erniebot/backends/bce.py | 8 ++++---- erniebot/src/erniebot/types.py | 14 ++------------ erniebot/tests/test_chat_completion_aio.py | 6 ++++-- 3 files changed, 10 insertions(+), 18 deletions(-) diff --git a/erniebot/src/erniebot/backends/bce.py b/erniebot/src/erniebot/backends/bce.py index c97d31a36..3e6f48ad9 100644 --- a/erniebot/src/erniebot/backends/bce.py +++ b/erniebot/src/erniebot/backends/bce.py @@ -75,8 +75,8 @@ def request( ) except (errors.TokenExpiredError, errors.InvalidTokenError): logging.warning( - "The access token provided is invalid or has expired. " - "An automatic update will be performed before retrying." + "The access token provided is invalid or has expired." + " An automatic update will be performed before retrying." ) access_token = self._auth_manager.update_auth_token() url_with_token = add_query_params(url, [("access_token", access_token)]) @@ -126,8 +126,8 @@ async def arequest( ) except (errors.TokenExpiredError, errors.InvalidTokenError): logging.warning( - "The access token provided is invalid or has expired. " - "An automatic update will be performed before retrying." + "The access token provided is invalid or has expired." + " An automatic update will be performed before retrying." ) # XXX: The default executor is used. access_token = await loop.run_in_executor(None, self._auth_manager.update_auth_token) diff --git a/erniebot/src/erniebot/types.py b/erniebot/src/erniebot/types.py index c52d1d043..78a5065d7 100644 --- a/erniebot/src/erniebot/types.py +++ b/erniebot/src/erniebot/types.py @@ -15,19 +15,9 @@ from __future__ import annotations from dataclasses import dataclass -from typing import ( - IO, - TYPE_CHECKING, - Any, - AsyncIterator, - Dict, - Iterator, - Optional, - TypeVar, -) +from typing import IO, Any, AsyncIterator, Dict, Iterator, Optional, TypeVar -if TYPE_CHECKING: - from typing_extensions import TypeAlias +from typing_extensions import TypeAlias from .response import EBResponse diff --git a/erniebot/tests/test_chat_completion_aio.py b/erniebot/tests/test_chat_completion_aio.py index 4b2e05da9..0ef8ef87b 100644 --- a/erniebot/tests/test_chat_completion_aio.py +++ b/erniebot/tests/test_chat_completion_aio.py @@ -78,6 +78,8 @@ async def test_chat_completion_aio(target, args): erniebot.api_type = "qianfan" - asyncio.run(test_chat_completion_aio(acreate_chat_completion, args=("ernie-turbo",))) + async def main(): + await test_chat_completion_aio(acreate_chat_completion, args=("ernie-turbo",)) + await test_chat_completion_aio(acreate_chat_completion_stream, args=("ernie-turbo",)) - asyncio.run(test_chat_completion_aio(acreate_chat_completion_stream, args=("ernie-turbo",))) + asyncio.run(main()) From 0a9e9e609ca78b6c7b57938fd9271a3ddc49871b Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Thu, 21 Dec 2023 18:09:42 +0800 Subject: [PATCH 03/63] Enhance file_io --- .../src/erniebot_agent/file_io/__init__.py | 5 +- .../src/erniebot_agent/file_io/base.py | 13 +- .../src/erniebot_agent/file_io/factory.py | 29 -- .../erniebot_agent/file_io/file_manager.py | 253 ++++++++++++------ .../erniebot_agent/file_io/file_registry.py | 55 ++-- .../file_io/global_file_manager.py | 73 +++++ .../src/erniebot_agent/file_io/local_file.py | 65 +++-- .../src/erniebot_agent/file_io/protocol.py | 7 +- .../src/erniebot_agent/file_io/remote_file.py | 64 +++-- 9 files changed, 373 insertions(+), 191 deletions(-) delete mode 100644 erniebot-agent/src/erniebot_agent/file_io/factory.py create mode 100644 erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py diff --git a/erniebot-agent/src/erniebot_agent/file_io/__init__.py b/erniebot-agent/src/erniebot_agent/file_io/__init__.py index 89807bda3..d4273b94b 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file_io/__init__.py @@ -12,4 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -from erniebot_agent.file_io.factory import get_file_manager +from erniebot_agent.file_io.global_file_manager import ( + configure_global_file_manager, + get_global_file_manager, +) diff --git a/erniebot-agent/src/erniebot_agent/file_io/base.py b/erniebot-agent/src/erniebot_agent/file_io/base.py index f1dcc1a8c..9508e6028 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/base.py +++ b/erniebot-agent/src/erniebot_agent/file_io/base.py @@ -13,7 +13,10 @@ # limitations under the License. import abc -from typing import Any, Dict +import os +from typing import Any, Dict, Union + +import anyio class File(metaclass=abc.ABCMeta): @@ -23,7 +26,7 @@ def __init__( id: str, filename: str, byte_size: int, - created_at: int, + created_at: str, purpose: str, metadata: Dict[str, Any], ) -> None: @@ -40,7 +43,7 @@ def __eq__(self, other: object) -> bool: if isinstance(other, File): return self.id == other.id else: - return False + return NotImplemented def __repr__(self) -> str: attrs_str = self._get_attrs_str() @@ -50,6 +53,10 @@ def __repr__(self) -> str: async def read_contents(self) -> bytes: raise NotImplementedError + async def write_contents_to(self, local_path: Union[str, os.PathLike[str]]) -> None: + contents = await self.read_contents() + await anyio.Path(local_path).write_bytes(contents) + def get_file_repr(self) -> str: return f"{self.id}" diff --git a/erniebot-agent/src/erniebot_agent/file_io/factory.py b/erniebot-agent/src/erniebot_agent/file_io/factory.py deleted file mode 100644 index 77f46cab1..000000000 --- a/erniebot-agent/src/erniebot_agent/file_io/factory.py +++ /dev/null @@ -1,29 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import functools -from typing import Optional - -from erniebot_agent.file_io.file_manager import FileManager -from erniebot_agent.file_io.remote_file import AIStudioFileClient - - -@functools.lru_cache(maxsize=None) -def get_file_manager(access_token: Optional[str] = None) -> FileManager: - if access_token is None: - # TODO: Use a default global access token. - return FileManager() - else: - remote_file_client = AIStudioFileClient(access_token=access_token) - return FileManager(remote_file_client) diff --git a/erniebot-agent/src/erniebot_agent/file_io/file_manager.py b/erniebot-agent/src/erniebot_agent/file_io/file_manager.py index 631d524ac..253a64ffa 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file_io/file_manager.py @@ -17,51 +17,51 @@ import pathlib import tempfile import uuid -import weakref -from typing import Any, Dict, List, Literal, Optional, Union, overload +from types import TracebackType +from typing import Any, Dict, List, Literal, Optional, Type, Union, final, overload import anyio -from typing_extensions import TypeAlias +from typing_extensions import Self, TypeAlias +from erniebot_agent.file_io import protocol from erniebot_agent.file_io.base import File -from erniebot_agent.file_io.file_registry import FileRegistry, get_file_registry +from erniebot_agent.file_io.file_registry import FileRegistry from erniebot_agent.file_io.local_file import LocalFile, create_local_file_from_path -from erniebot_agent.file_io.protocol import FilePurpose from erniebot_agent.file_io.remote_file import RemoteFile, RemoteFileClient -from erniebot_agent.utils.exception import FileError +from erniebot_agent.utils.exceptions import FileError +from erniebot_agent.utils.mixins import Closeable logger = logging.getLogger(__name__) -FilePath: TypeAlias = Union[str, os.PathLike] +FilePath: TypeAlias = Union[str, os.PathLike[str]] -class FileManager(object): - _remote_file_client: Optional[RemoteFileClient] +@final +class FileManager(Closeable): + _temp_dir: Optional[tempfile.TemporaryDirectory] = None def __init__( self, remote_file_client: Optional[RemoteFileClient] = None, - *, - auto_register: bool = True, save_dir: Optional[FilePath] = None, + *, + prune_on_close: bool = True, ) -> None: super().__init__() - if remote_file_client is not None: - self._remote_file_client = remote_file_client - else: - self._remote_file_client = None - self._auto_register = auto_register + + self._remote_file_client = remote_file_client if save_dir is not None: self._save_dir = pathlib.Path(save_dir) else: # This can be done lazily, but we need to be careful about race conditions. - self._save_dir = self._fs_create_temp_dir() + self._temp_dir = self._create_temp_dir() + self._save_dir = pathlib.Path(self._temp_dir.name) + self._prune_on_close = prune_on_close - self._file_registry = get_file_registry() + self._file_registry = FileRegistry() + self._fully_managed_files: List[Union[LocalFile, RemoteFile]] = [] - @property - def registry(self) -> FileRegistry: - return self._file_registry + self._closed = False @property def remote_file_client(self) -> RemoteFileClient: @@ -70,14 +70,29 @@ def remote_file_client(self) -> RemoteFileClient: else: return self._remote_file_client + @property + def closed(self): + return self._closed + + async def __aenter__(self) -> Self: + return self + + async def __aexit__( + self, + exc_type: Optional[Type[BaseException]] = None, + exc_value: Optional[BaseException] = None, + traceback: Optional[TracebackType] = None, + ) -> None: + await self.close() + @overload async def create_file_from_path( self, file_path: FilePath, *, - file_purpose: FilePurpose = ..., + file_purpose: protocol.FilePurpose = ..., file_metadata: Optional[Dict[str, Any]] = ..., - file_type: Literal["local"] = ..., + file_type: Literal["local"], ) -> LocalFile: ... @@ -86,26 +101,35 @@ async def create_file_from_path( self, file_path: FilePath, *, - file_purpose: FilePurpose = ..., + file_purpose: protocol.FilePurpose = ..., file_metadata: Optional[Dict[str, Any]] = ..., file_type: Literal["remote"], ) -> RemoteFile: ... + @overload async def create_file_from_path( self, file_path: FilePath, *, - file_purpose: FilePurpose = "assistants", + file_purpose: protocol.FilePurpose = ..., + file_metadata: Optional[Dict[str, Any]] = ..., + file_type: None = ..., + ) -> Union[LocalFile, RemoteFile]: + ... + + async def create_file_from_path( + self, + file_path: FilePath, + *, + file_purpose: protocol.FilePurpose = "assistants", file_metadata: Optional[Dict[str, Any]] = None, file_type: Optional[Literal["local", "remote"]] = None, ) -> Union[LocalFile, RemoteFile]: + self.ensure_not_closed() file: Union[LocalFile, RemoteFile] if file_type is None: - if self._remote_file_client is not None: - file_type = "remote" - else: - file_type = "local" + file_type = self._get_default_file_type() if file_type == "local": file = await self.create_local_file_from_path(file_path, file_purpose, file_metadata) elif file_type == "remote": @@ -117,10 +141,10 @@ async def create_file_from_path( async def create_local_file_from_path( self, file_path: FilePath, - file_purpose: FilePurpose, + file_purpose: protocol.FilePurpose, file_metadata: Optional[Dict[str, Any]], ) -> LocalFile: - file = create_local_file_from_path( + file = await self._create_local_file_from_path( pathlib.Path(file_path), file_purpose, file_metadata or {}, @@ -129,13 +153,18 @@ async def create_local_file_from_path( return file async def create_remote_file_from_path( - self, file_path: FilePath, file_purpose: FilePurpose, file_metadata: Optional[Dict[str, Any]] + self, + file_path: FilePath, + file_purpose: protocol.FilePurpose, + file_metadata: Optional[Dict[str, Any]], ) -> RemoteFile: - file = await self.remote_file_client.upload_file( - pathlib.Path(file_path), file_purpose, file_metadata or {} + file = await self._create_remote_file_from_path( + pathlib.Path(file_path), + file_purpose, + file_metadata, ) - if self._auto_register: - self._file_registry.register_file(file) + self._file_registry.register_file(file) + self._fully_managed_files.append(file) return file @overload @@ -144,9 +173,9 @@ async def create_file_from_bytes( file_contents: bytes, filename: str, *, - file_purpose: FilePurpose = ..., + file_purpose: protocol.FilePurpose = ..., file_metadata: Optional[Dict[str, Any]] = ..., - file_type: Literal["local"] = ..., + file_type: Literal["local"], ) -> LocalFile: ... @@ -156,80 +185,152 @@ async def create_file_from_bytes( file_contents: bytes, filename: str, *, - file_purpose: FilePurpose = ..., + file_purpose: protocol.FilePurpose = ..., file_metadata: Optional[Dict[str, Any]] = ..., file_type: Literal["remote"], ) -> RemoteFile: ... + @overload + async def create_file_from_bytes( + self, + file_contents: bytes, + filename: str, + *, + file_purpose: protocol.FilePurpose = ..., + file_metadata: Optional[Dict[str, Any]] = ..., + file_type: None = ..., + ) -> Union[LocalFile, RemoteFile]: + ... + async def create_file_from_bytes( self, file_contents: bytes, filename: str, *, - file_purpose: FilePurpose = "assistants", + file_purpose: protocol.FilePurpose = "assistants", file_metadata: Optional[Dict[str, Any]] = None, file_type: Optional[Literal["local", "remote"]] = None, ) -> Union[LocalFile, RemoteFile]: - # Can we do this with in-memory files? - file_path = await self._fs_create_file( - prefix=pathlib.PurePath(filename).stem, suffix=pathlib.PurePath(filename).suffix + self.ensure_not_closed() + if file_type is None: + file_type = self._get_default_file_type() + file_path = self._get_unique_file_path( + prefix=pathlib.PurePath(filename).stem, + suffix=pathlib.PurePath(filename).suffix, ) + async_file_path = anyio.Path(file_path) + await async_file_path.touch() + should_remove_file = True try: - async with await file_path.open("wb") as f: + async with await async_file_path.open("wb") as f: await f.write(file_contents) - if file_type is None: - if self._remote_file_client is not None: - file_type = "remote" - else: - file_type = "local" - file = await self.create_file_from_path( - file_path, - file_purpose=file_purpose, - file_metadata=file_metadata, - file_type=file_type, - ) + file: Union[LocalFile, RemoteFile] + if file_type == "local": + file = await self._create_local_file_from_path(file_path, file_purpose, file_metadata) + should_remove_file = False + elif file_type == "remote": + file = await self._create_remote_file_from_path( + file_path, + file_purpose, + file_metadata, + ) + else: + raise ValueError(f"Unsupported file type: {file_type}") finally: - if file_type == "remote": - await file_path.unlink() + if should_remove_file: + await async_file_path.unlink() + self._file_registry.register_file(file) + self._fully_managed_files.append(file) return file async def retrieve_remote_file_by_id(self, file_id: str) -> RemoteFile: + self.ensure_not_closed() file = await self.remote_file_client.retrieve_file(file_id) - if self._auto_register: - self._file_registry.register_file(file, allow_overwrite=True) + self._file_registry.register_file(file) return file + async def list_remote_files(self) -> List[RemoteFile]: + self.ensure_not_closed() + files = await self.remote_file_client.list_files() + return files + def look_up_file_by_id(self, file_id: str) -> Optional[File]: + self.ensure_not_closed() file = self._file_registry.look_up_file(file_id) if file is None: raise FileError( - f"File with ID '{file_id}' not found. " - "Please check if the file exists and the `file_id` is correct." + f"File with ID {repr(file_id)} not found. " + "Please check if `file_id` is correct and the file is registered." ) return file - async def list_remote_files(self) -> List[RemoteFile]: - files = await self.remote_file_client.list_files() - if self._auto_register: - for file in files: - self._file_registry.register_file(file, allow_overwrite=True) - return files + def list_registered_files(self) -> List[File]: + self.ensure_not_closed() + return self._file_registry.list_files() + + async def prune(self) -> None: + for file in self._fully_managed_files: + if isinstance(file, RemoteFile): + # FIXME: Currently this is not supported. + # await file.delete() + pass + elif isinstance(file, LocalFile): + assert self._save_dir in file.path.parents + await anyio.Path(file.path).unlink() + else: + raise AssertionError("Unexpected file type") + self._file_registry.unregister_file(file) + self._fully_managed_files.clear() - async def _fs_create_file( + async def close(self) -> None: + if not self._closed: + if self._remote_file_client is not None: + await self._remote_file_client.close() + if self._prune_on_close: + await self.prune() + if self._temp_dir is not None: + self._clean_up_temp_dir(self._temp_dir) + self._closed = True + + async def _create_local_file_from_path( + self, + file_path: pathlib.Path, + file_purpose: protocol.FilePurpose, + file_metadata: Optional[Dict[str, Any]], + ) -> LocalFile: + return create_local_file_from_path( + pathlib.Path(file_path), + file_purpose, + file_metadata or {}, + ) + + async def _create_remote_file_from_path( + self, + file_path: pathlib.Path, + file_purpose: protocol.FilePurpose, + file_metadata: Optional[Dict[str, Any]], + ) -> RemoteFile: + file = await self.remote_file_client.upload_file(file_path, file_purpose, file_metadata or {}) + return file + + def _get_default_file_type(self) -> Literal["local", "remote"]: + if self._remote_file_client is not None: + return "remote" + else: + return "local" + + def _get_unique_file_path( self, prefix: Optional[str] = None, suffix: Optional[str] = None - ) -> anyio.Path: + ) -> pathlib.Path: filename = f"{prefix or ''}{str(uuid.uuid4())}{suffix or ''}" - file_path = anyio.Path(self._save_dir / filename) - await file_path.touch() + file_path = self._save_dir / filename return file_path - def _fs_create_temp_dir(self) -> pathlib.Path: + @staticmethod + def _create_temp_dir() -> tempfile.TemporaryDirectory: temp_dir = tempfile.TemporaryDirectory() - # The temporary directory shall be cleaned up when the file manager is - # garbage collected. - weakref.finalize(self, self._clean_up_temp_dir, temp_dir) - return pathlib.Path(temp_dir.name) + return temp_dir @staticmethod def _clean_up_temp_dir(temp_dir: tempfile.TemporaryDirectory) -> None: diff --git a/erniebot-agent/src/erniebot_agent/file_io/file_registry.py b/erniebot-agent/src/erniebot_agent/file_io/file_registry.py index 9e21badc3..607562db8 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/file_registry.py +++ b/erniebot-agent/src/erniebot_agent/file_io/file_registry.py @@ -12,44 +12,49 @@ # See the License for the specific language governing permissions and # limitations under the License. -import threading -from typing import Dict, List, Optional +from typing import Dict, List, Optional, final from erniebot_agent.file_io.base import File -from erniebot_agent.utils.misc import Singleton -class FileRegistry(metaclass=Singleton): +class BaseFileRegistry(object): + def register_file(self, file: File, *, allow_overwrite: bool = False, check_type: bool = True) -> None: + raise NotImplementedError + + def unregister_file(self, file: File) -> None: + raise NotImplementedError + + def look_up_file(self, file_id: str) -> Optional[File]: + raise NotImplementedError + + def list_files(self) -> List[File]: + raise NotImplementedError + + +@final +class FileRegistry(BaseFileRegistry): def __init__(self) -> None: super().__init__() self._id_to_file: Dict[str, File] = {} - self._lock = threading.Lock() - def register_file(self, file: File, *, allow_overwrite: bool = False) -> None: + def register_file(self, file: File, *, allow_overwrite: bool = False, check_type: bool = True) -> None: file_id = file.id - with self._lock: - if not allow_overwrite and file_id in self._id_to_file: - raise RuntimeError(f"ID {repr(file_id)} is already registered.") - self._id_to_file[file_id] = file + if file_id in self._id_to_file: + if not allow_overwrite: + raise RuntimeError(f"File with ID {repr(file_id)} is already registered.") + else: + if check_type and type(file) is not type(self._id_to_file[file_id]): # noqa: E721 + raise RuntimeError("Cannot register a file with a different type.") + self._id_to_file[file_id] = file def unregister_file(self, file: File) -> None: file_id = file.id - with self._lock: - if file_id not in self._id_to_file: - raise RuntimeError(f"ID {repr(file_id)} is not registered.") - self._id_to_file.pop(file_id) + if file_id not in self._id_to_file: + raise RuntimeError(f"File with ID {repr(file_id)} is not registered.") + self._id_to_file.pop(file_id) def look_up_file(self, file_id: str) -> Optional[File]: - with self._lock: - return self._id_to_file.get(file_id, None) + return self._id_to_file.get(file_id, None) def list_files(self) -> List[File]: - with self._lock: - return list(self._id_to_file.values()) - - -_file_registry = FileRegistry() - - -def get_file_registry() -> FileRegistry: - return _file_registry + return list(self._id_to_file.values()) diff --git a/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py b/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py new file mode 100644 index 000000000..caeafb935 --- /dev/null +++ b/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py @@ -0,0 +1,73 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import asyncio +import atexit +from typing import Any, List, Optional + +from erniebot_agent.file_io.file_manager import FileManager +from erniebot_agent.file_io.remote_file import AIStudioFileClient +from erniebot_agent.utils import config_from_environ as C +from erniebot_agent.utils.mixins import Closeable + +_global_file_manager: Optional[FileManager] = None +_objects_to_close: List[Closeable] = [] + + +def get_global_file_manager() -> FileManager: + global _global_file_manager + if _global_file_manager is None: + _global_file_manager = _create_default_file_manager(access_token=None, save_dir=None) + return _global_file_manager + + +def configure_global_file_manager( + access_token: Optional[str] = None, save_dir: Optional[str] = None, **opts: Any +) -> None: + global _global_file_manager + if _global_file_manager is not None: + raise RuntimeError("The global file manager can only be configured once.") + _global_file_manager = _create_default_file_manager(access_token=access_token, save_dir=save_dir, **opts) + + +def _create_default_file_manager( + access_token: Optional[str], save_dir: Optional[str], **opts: Any +) -> FileManager: + if access_token is None: + access_token = C.get_global_access_token() + if save_dir is None: + save_dir = C.get_global_save_dir() + if access_token is not None: + remote_file_client = AIStudioFileClient(access_token=access_token) + else: + remote_file_client = None + file_manager = FileManager(remote_file_client, save_dir, **opts) + _objects_to_close.append(file_manager) + return file_manager + + +def _close_objects(): + async def _close_objects_sequentially(): + for obj in _objects_to_close: + await obj.close() + + if _objects_to_close: + # Since async atexit is not officially supported by Python, + # we start a new event loop to do the cleanup. + asyncio.run(_close_objects_sequentially()) + _objects_to_close.clear() + + +# FIXME: The exit handler may not be called when using multiprocessing. +atexit.register(_close_objects) diff --git a/erniebot-agent/src/erniebot_agent/file_io/local_file.py b/erniebot-agent/src/erniebot_agent/file_io/local_file.py index 9b2b4ef5e..b992105b0 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/local_file.py +++ b/erniebot-agent/src/erniebot_agent/file_io/local_file.py @@ -13,18 +13,36 @@ # limitations under the License. import pathlib -import time import uuid from typing import Any, Dict import anyio +from erniebot_agent.file_io import protocol from erniebot_agent.file_io.base import File -from erniebot_agent.file_io.protocol import ( - FilePurpose, - build_local_file_id_from_uuid, - is_local_file_id, -) + + +def create_local_file_from_path( + file_path: pathlib.Path, + file_purpose: protocol.FilePurpose, + file_metadata: Dict[str, Any], +) -> "LocalFile": + if not file_path.exists(): + raise FileNotFoundError(f"File {file_path} does not exist.") + file_id = _generate_local_file_id() + filename = file_path.name + byte_size = file_path.stat().st_size + created_at = protocol.get_timestamp() + file = LocalFile( + id=file_id, + filename=filename, + byte_size=byte_size, + created_at=created_at, + purpose=file_purpose, + metadata=file_metadata, + path=file_path, + ) + return file class LocalFile(File): @@ -34,13 +52,15 @@ def __init__( id: str, filename: str, byte_size: int, - created_at: int, - purpose: FilePurpose, + created_at: str, + purpose: protocol.FilePurpose, metadata: Dict[str, Any], path: pathlib.Path, + validate_file_id: bool = True, ) -> None: - if not is_local_file_id(id): - raise ValueError(f"Invalid file ID: {id}") + if validate_file_id: + if not protocol.is_local_file_id(id): + raise ValueError(f"Invalid file ID: {id}") super().__init__( id=id, filename=filename, @@ -60,28 +80,5 @@ def _get_attrs_str(self) -> str: return attrs_str -def create_local_file_from_path( - file_path: pathlib.Path, - file_purpose: FilePurpose, - file_metadata: Dict[str, Any], -) -> LocalFile: - if not file_path.exists(): - raise FileNotFoundError(f"File {file_path} does not exist.") - file_id = _generate_local_file_id() - filename = file_path.name - byte_size = file_path.stat().st_size - created_at = int(time.time()) - file = LocalFile( - id=file_id, - filename=filename, - byte_size=byte_size, - created_at=created_at, - purpose=file_purpose, - metadata=file_metadata, - path=file_path, - ) - return file - - def _generate_local_file_id(): - return build_local_file_id_from_uuid(str(uuid.uuid1())) + return protocol.create_local_file_id_from_uuid(str(uuid.uuid1())) diff --git a/erniebot-agent/src/erniebot_agent/file_io/protocol.py b/erniebot-agent/src/erniebot_agent/file_io/protocol.py index 4b6e28d5d..f636dfe0f 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/protocol.py +++ b/erniebot-agent/src/erniebot_agent/file_io/protocol.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import datetime import re from typing import List, Literal @@ -28,10 +29,14 @@ _compiled_remote_file_id_pattern = re.compile(_REMOTE_FILE_ID_PATTERN) -def build_local_file_id_from_uuid(uuid: str) -> str: +def create_local_file_id_from_uuid(uuid: str) -> str: return _LOCAL_FILE_ID_PREFIX + uuid +def get_timestamp() -> str: + return datetime.datetime.now().isoformat(sep=" ", timespec="seconds") + + def is_file_id(str_: str) -> bool: return is_local_file_id(str_) or is_remote_file_id(str_) diff --git a/erniebot-agent/src/erniebot_agent/file_io/remote_file.py b/erniebot-agent/src/erniebot_agent/file_io/remote_file.py index f48335350..02d499173 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/remote_file.py +++ b/erniebot-agent/src/erniebot_agent/file_io/remote_file.py @@ -21,9 +21,10 @@ import aiohttp +from erniebot_agent.file_io import protocol from erniebot_agent.file_io.base import File -from erniebot_agent.file_io.protocol import FilePurpose, is_remote_file_id -from erniebot_agent.utils.exception import FileError +from erniebot_agent.utils.exceptions import FileError +from erniebot_agent.utils.mixins import Closeable class RemoteFile(File): @@ -33,13 +34,15 @@ def __init__( id: str, filename: str, byte_size: int, - created_at: int, - purpose: FilePurpose, + created_at: str, + purpose: protocol.FilePurpose, metadata: Dict[str, Any], client: "RemoteFileClient", + validate_file_id: bool = True, ) -> None: - if not is_remote_file_id(id): - raise FileError(f"Invalid file ID: {id}") + if validate_file_id: + if not protocol.is_remote_file_id(id): + raise FileError(f"Invalid file ID: {id}") super().__init__( id=id, filename=filename, @@ -50,6 +53,10 @@ def __init__( ) self._client = client + @property + def client(self) -> "RemoteFileClient": + return self._client + async def read_contents(self) -> bytes: file_contents = await self._client.retrieve_file_contents(self.id) return file_contents @@ -64,10 +71,10 @@ def get_file_repr_with_url(self, url: str) -> str: return f"{self.get_file_repr()}{url}" -class RemoteFileClient(metaclass=abc.ABCMeta): +class RemoteFileClient(Closeable, metaclass=abc.ABCMeta): @abc.abstractmethod async def upload_file( - self, file_path: pathlib.Path, file_purpose: FilePurpose, file_metadata: Dict[str, Any] + self, file_path: pathlib.Path, file_purpose: protocol.FilePurpose, file_metadata: Dict[str, Any] ) -> RemoteFile: raise NotImplementedError @@ -104,11 +111,19 @@ def __init__( ) -> None: super().__init__() self._access_token = access_token + if aiohttp_session is None: + aiohttp_session = self._create_aiohttp_session() self._session = aiohttp_session + self._closed = False + + @property + def closed(self) -> bool: + return self._closed async def upload_file( - self, file_path: pathlib.Path, file_purpose: FilePurpose, file_metadata: Dict[str, Any] + self, file_path: pathlib.Path, file_purpose: protocol.FilePurpose, file_metadata: Dict[str, Any] ) -> RemoteFile: + self.ensure_not_closed() url = self._get_url(self._UPLOAD_ENDPOINT) headers: Dict[str, str] = {} headers.update(self._get_default_headers()) @@ -125,9 +140,10 @@ async def upload_file( raise_for_status=True, ) result = self._get_result_from_response_body(resp_bytes) - return self._build_file_obj_from_dict(result) + return self._create_file_obj_from_dict(result) async def retrieve_file(self, file_id: str) -> RemoteFile: + self.ensure_not_closed() url = self._get_url(self._RETRIEVE_ENDPOINT).format(file_id=file_id) headers: Dict[str, str] = {} headers.update(self._get_default_headers()) @@ -138,9 +154,10 @@ async def retrieve_file(self, file_id: str) -> RemoteFile: raise_for_status=True, ) result = self._get_result_from_response_body(resp_bytes) - return self._build_file_obj_from_dict(result) + return self._create_file_obj_from_dict(result) async def retrieve_file_contents(self, file_id: str) -> bytes: + self.ensure_not_closed() url = self._get_url(self._RETRIEVE_CONTENTS_ENDPOINT).format(file_id=file_id) headers: Dict[str, str] = {} headers.update(self._get_default_headers()) @@ -153,6 +170,7 @@ async def retrieve_file_contents(self, file_id: str) -> bytes: return resp_bytes async def list_files(self) -> List[RemoteFile]: + self.ensure_not_closed() url = self._get_url(self._LIST_ENDPOINT) headers: Dict[str, str] = {} headers.update(self._get_default_headers()) @@ -165,7 +183,7 @@ async def list_files(self) -> List[RemoteFile]: result = self._get_result_from_response_body(resp_bytes) files: List[RemoteFile] = [] for item in result: - file = self._build_file_obj_from_dict(item) + file = self._create_file_obj_from_dict(item) files.append(file) return files @@ -186,14 +204,12 @@ async def create_temporary_url(self, file_id: str, expire_after: float) -> str: result = self._get_result_from_response_body(resp_bytes) return result["fileUrl"] - async def _request(self, *args: Any, **kwargs: Any) -> bytes: - if self._session is not None: - async with self._session.request(*args, **kwargs) as response: - return await response.read() - else: - async with aiohttp.ClientSession(**self._get_session_config()) as session: - async with session.request(*args, **kwargs) as response: - return await response.read() + async def close(self) -> None: + if not self._closed: + await self._session.close() + + def _create_aiohttp_session(self) -> aiohttp.ClientSession: + return aiohttp.ClientSession(**self._get_session_config()) def _get_session_config(self) -> Dict[str, Any]: return {} @@ -203,9 +219,13 @@ def _get_default_headers(self) -> Dict[str, str]: "Authorization": f"token {self._access_token}", } - def _build_file_obj_from_dict(self, dict_: Dict[str, Any]) -> RemoteFile: + async def _request(self, *args: Any, **kwargs: Any) -> bytes: + async with self._session.request(*args, **kwargs) as response: + return await response.read() + + def _create_file_obj_from_dict(self, dict_: Dict[str, Any]) -> RemoteFile: metadata: Dict[str, Any] - if "meta" in dict_: + if dict_.get("meta"): metadata = json.loads(dict_["meta"]) if not isinstance(metadata, dict): raise FileError(f"Invalid metadata: {dict_['meta']}") From 38cb8ff3b12f1a541341b6c922e6de63b70473a2 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Thu, 21 Dec 2023 18:09:58 +0800 Subject: [PATCH 04/63] Update library code --- .../src/erniebot_agent/agents/base.py | 32 ++++++------- .../callback/handlers/logging_handler.py | 3 +- .../src/erniebot_agent/agents/schema.py | 6 ++- .../erniebot_agent/chat_models/erniebot.py | 3 ++ .../langchain/chat_models/erniebot.py | 4 +- .../extensions/langchain/embeddings/ernie.py | 4 +- .../extensions/langchain/llms/erniebot.py | 4 +- .../retrieval/baizhong_search.py | 2 +- .../src/erniebot_agent/tools/remote_tool.py | 3 +- .../erniebot_agent/tools/remote_toolkit.py | 32 +++++++++---- .../src/erniebot_agent/tools/schema.py | 2 +- .../src/erniebot_agent/tools/utils.py | 2 +- .../utils/config_from_environ.py | 36 +++++++++++++++ .../utils/{exception.py => exceptions.py} | 4 ++ .../src/erniebot_agent/utils/logging.py | 19 +++----- .../utils/{gradio_mixin.py => mixins.py} | 46 ++++++++++++++++--- .../src/erniebot_agent/utils/temp_file.py | 39 ++++++++++++++++ 17 files changed, 180 insertions(+), 61 deletions(-) create mode 100644 erniebot-agent/src/erniebot_agent/utils/config_from_environ.py rename erniebot-agent/src/erniebot_agent/utils/{exception.py => exceptions.py} (95%) rename erniebot-agent/src/erniebot_agent/utils/{gradio_mixin.py => mixins.py} (85%) create mode 100644 erniebot-agent/src/erniebot_agent/utils/temp_file.py diff --git a/erniebot-agent/src/erniebot_agent/agents/base.py b/erniebot-agent/src/erniebot_agent/agents/base.py index 630767a83..4ec89aebb 100644 --- a/erniebot-agent/src/erniebot_agent/agents/base.py +++ b/erniebot-agent/src/erniebot_agent/agents/base.py @@ -14,7 +14,8 @@ import abc import json -from typing import Any, Dict, List, Literal, Optional, Union +import logging +from typing import Any, Dict, List, Literal, Optional, Union, final from erniebot_agent import file_io from erniebot_agent.agents.callback.callback_manager import CallbackManager @@ -27,15 +28,16 @@ ToolResponse, ) from erniebot_agent.chat_models.base import ChatModel +from erniebot_agent.file_io import protocol from erniebot_agent.file_io.base import File from erniebot_agent.file_io.file_manager import FileManager -from erniebot_agent.file_io.protocol import is_local_file_id, is_remote_file_id from erniebot_agent.memory.base import Memory from erniebot_agent.messages import Message, SystemMessage from erniebot_agent.tools.base import BaseTool from erniebot_agent.tools.tool_manager import ToolManager -from erniebot_agent.utils.gradio_mixin import GradioMixin -from erniebot_agent.utils.logging import logger +from erniebot_agent.utils.mixins import GradioMixin + +logger = logging.getLogger(__name__) class BaseAgent(metaclass=abc.ABCMeta): @@ -76,7 +78,7 @@ def __init__( else: self._callback_manager = CallbackManager(callbacks) if file_manager is None: - file_manager = file_io.get_file_manager() + file_manager = file_io.get_global_file_manager() self.plugins = plugins self._file_manager = file_manager self._init_file_repr() @@ -94,6 +96,7 @@ def _init_file_repr(self): def tools(self) -> List[BaseTool]: return self._tool_manager.get_tools() + @final async def async_run(self, prompt: str, files: Optional[List[File]] = None) -> AgentResponse: await self._callback_manager.on_run_start(agent=self, prompt=prompt) agent_resp = await self._async_run(prompt, files) @@ -113,6 +116,7 @@ def reset_memory(self) -> None: async def _async_run(self, prompt: str, files: Optional[List[File]] = None) -> AgentResponse: raise NotImplementedError + @final async def _async_run_tool(self, tool_name: str, tool_args: str) -> ToolResponse: tool = self._tool_manager.get_tool(tool_name) await self._callback_manager.on_tool_start(agent=self, tool=tool, input_args=tool_args) @@ -124,6 +128,7 @@ async def _async_run_tool(self, tool_name: str, tool_args: str) -> ToolResponse: await self._callback_manager.on_tool_end(agent=self, tool=tool, response=tool_resp) return tool_resp + @final async def _async_run_llm(self, messages: List[Message], **opts: Any) -> LLMResponse: await self._callback_manager.on_llm_start(agent=self, llm=self.llm, messages=messages) try: @@ -170,16 +175,7 @@ async def _sniff_and_extract_files_from_args( agent_files: List[AgentFile] = [] for val in args.values(): if isinstance(val, str): - if is_local_file_id(val): - if self._file_manager is None: - logger.warning( - f"A file is used by {repr(tool)}, but the agent has no file manager to fetch it." - ) - continue - file = self._file_manager.look_up_file_by_id(val) - if file is None: - raise RuntimeError(f"Unregistered ID {repr(val)} is used by {repr(tool)}.") - elif is_remote_file_id(val): + if protocol.is_file_id(val): if self._file_manager is None: logger.warning( f"A file is used by {repr(tool)}, but the agent has no file manager to fetch it." @@ -187,10 +183,8 @@ async def _sniff_and_extract_files_from_args( continue file = self._file_manager.look_up_file_by_id(val) if file is None: - file = await self._file_manager.retrieve_remote_file_by_id(val) - else: - continue - agent_files.append(AgentFile(file=file, type=file_type, used_by=tool.tool_name)) + raise RuntimeError(f"Unregistered file with ID {repr(val)} is used by {repr(tool)}.") + agent_files.append(AgentFile(file=file, type=file_type, used_by=tool.tool_name)) elif isinstance(val, dict): agent_files.extend(await self._sniff_and_extract_files_from_args(val, tool, file_type)) elif isinstance(val, list) and len(val) > 0 and isinstance(val[0], dict): diff --git a/erniebot-agent/src/erniebot_agent/agents/callback/handlers/logging_handler.py b/erniebot-agent/src/erniebot_agent/agents/callback/handlers/logging_handler.py index e712f99d0..9f980cabd 100644 --- a/erniebot-agent/src/erniebot_agent/agents/callback/handlers/logging_handler.py +++ b/erniebot-agent/src/erniebot_agent/agents/callback/handlers/logging_handler.py @@ -23,9 +23,10 @@ from erniebot_agent.messages import Message from erniebot_agent.tools.base import BaseTool from erniebot_agent.utils.json import to_pretty_json -from erniebot_agent.utils.logging import logger as default_logger from erniebot_agent.utils.output_style import ColoredContent +default_logger = logging.getLogger(__name__) + if TYPE_CHECKING: from erniebot_agent.agents.base import Agent diff --git a/erniebot-agent/src/erniebot_agent/agents/schema.py b/erniebot-agent/src/erniebot_agent/agents/schema.py index eaf6d73cd..75390bd26 100644 --- a/erniebot-agent/src/erniebot_agent/agents/schema.py +++ b/erniebot-agent/src/erniebot_agent/agents/schema.py @@ -18,8 +18,8 @@ from dataclasses import dataclass from typing import Dict, List, Literal, Optional, Tuple, Union +from erniebot_agent.file_io import protocol from erniebot_agent.file_io.base import File -from erniebot_agent.file_io.protocol import extract_file_ids from erniebot_agent.messages import AIMessage, Message @@ -80,6 +80,8 @@ def get_output_files(self) -> List[File]: return [agent_file.file for agent_file in self.files if agent_file.type == "output"] def get_tool_input_output_files(self, tool_name: str) -> Tuple[List[File], List[File]]: + # XXX: If a tool is used mutliple times, all related files will be + # returned in flattened lists. input_files: List[File] = [] output_files: List[File] = [] for agent_file in self.files: @@ -94,7 +96,7 @@ def get_tool_input_output_files(self, tool_name: str) -> Tuple[List[File], List[ def output_dict(self) -> Dict[str, List]: # 1. split the text into parts and add file id to each part - file_ids = extract_file_ids(self.text) + file_ids = protocol.extract_file_ids(self.text) places = [] for file_id in file_ids: diff --git a/erniebot-agent/src/erniebot_agent/chat_models/erniebot.py b/erniebot-agent/src/erniebot_agent/chat_models/erniebot.py index 6602c0692..4474082a8 100644 --- a/erniebot-agent/src/erniebot_agent/chat_models/erniebot.py +++ b/erniebot-agent/src/erniebot_agent/chat_models/erniebot.py @@ -36,6 +36,7 @@ Message, SeachInfo, ) +from erniebot_agent.utils import config_from_environ as C _T = TypeVar("_T", AIMessage, AIMessageChunk) @@ -61,6 +62,8 @@ def __init__( super().__init__(model=model, **default_chat_kwargs) self.api_type = api_type + if access_token is None: + access_token = C.get_global_access_token() self.access_token = access_token self.enable_multi_step_json = json.dumps( {"multi_step_tool_call_close": not enable_multi_step_tool_call} diff --git a/erniebot-agent/src/erniebot_agent/extensions/langchain/chat_models/erniebot.py b/erniebot-agent/src/erniebot_agent/extensions/langchain/chat_models/erniebot.py index 1d29ff5cf..92b026b6c 100644 --- a/erniebot-agent/src/erniebot_agent/extensions/langchain/chat_models/erniebot.py +++ b/erniebot-agent/src/erniebot_agent/extensions/langchain/chat_models/erniebot.py @@ -28,7 +28,7 @@ class ErnieBotChat(BaseChatModel): """ERNIE Bot Chat large language models API. To use, you should have the ``erniebot`` python package installed, and the - environment variable ``EB_ACCESS_TOKEN`` set with your AI Studio access token. + environment variable ``AISTUDIO_ACCESS_TOKEN`` set with your AI Studio access token. Example: .. code-block:: python @@ -96,7 +96,7 @@ def validate_enviroment(cls, values: Dict) -> Dict: values["aistudio_access_token"] = get_from_dict_or_env( values, "aistudio_access_token", - "EB_ACCESS_TOKEN", + "AISTUDIO_ACCESS_TOKEN", ) try: diff --git a/erniebot-agent/src/erniebot_agent/extensions/langchain/embeddings/ernie.py b/erniebot-agent/src/erniebot_agent/extensions/langchain/embeddings/ernie.py index 0844a0d48..3d5a662d3 100644 --- a/erniebot-agent/src/erniebot_agent/extensions/langchain/embeddings/ernie.py +++ b/erniebot-agent/src/erniebot_agent/extensions/langchain/embeddings/ernie.py @@ -11,7 +11,7 @@ class ErnieEmbeddings(BaseModel, Embeddings): """ERNIE embedding models. To use, you should have the ``erniebot`` python package installed, and the - environment variable ``EB_ACCESS_TOKEN`` set with your AI Studio access token. + environment variable ``AISTUDIO_ACCESS_TOKEN`` set with your AI Studio access token. Example: .. code-block:: python @@ -41,7 +41,7 @@ def validate_environment(cls, values: Dict) -> Dict: values["aistudio_access_token"] = get_from_dict_or_env( values, "aistudio_access_token", - "EB_ACCESS_TOKEN", + "AISTUDIO_ACCESS_TOKEN", ) try: diff --git a/erniebot-agent/src/erniebot_agent/extensions/langchain/llms/erniebot.py b/erniebot-agent/src/erniebot_agent/extensions/langchain/llms/erniebot.py index bb5402aa1..ed4a75c24 100644 --- a/erniebot-agent/src/erniebot_agent/extensions/langchain/llms/erniebot.py +++ b/erniebot-agent/src/erniebot_agent/extensions/langchain/llms/erniebot.py @@ -17,7 +17,7 @@ class ErnieBot(LLM): """ERNIE Bot large language models. To use, you should have the ``erniebot`` python package installed, and the - environment variable ``EB_ACCESS_TOKEN`` set with your AI Studio access token. + environment variable ``AISTUDIO_ACCESS_TOKEN`` set with your AI Studio access token. Example: .. code-block:: python @@ -82,7 +82,7 @@ def validate_enviroment(cls, values: Dict) -> Dict: values["aistudio_access_token"] = get_from_dict_or_env( values, "aistudio_access_token", - "EB_ACCESS_TOKEN", + "AISTUDIO_ACCESS_TOKEN", ) try: diff --git a/erniebot-agent/src/erniebot_agent/retrieval/baizhong_search.py b/erniebot-agent/src/erniebot_agent/retrieval/baizhong_search.py index b68a3ef7f..b984491bf 100644 --- a/erniebot-agent/src/erniebot_agent/retrieval/baizhong_search.py +++ b/erniebot-agent/src/erniebot_agent/retrieval/baizhong_search.py @@ -5,7 +5,7 @@ import requests -from erniebot_agent.utils.exception import BaizhongError +from erniebot_agent.utils.exceptions import BaizhongError logger = logging.getLogger(__name__) diff --git a/erniebot-agent/src/erniebot_agent/tools/remote_tool.py b/erniebot-agent/src/erniebot_agent/tools/remote_tool.py index 80a32d1e7..5096188e2 100644 --- a/erniebot-agent/src/erniebot_agent/tools/remote_tool.py +++ b/erniebot-agent/src/erniebot_agent/tools/remote_tool.py @@ -18,7 +18,7 @@ tool_response_contains_file, ) from erniebot_agent.utils.common import is_json_response -from erniebot_agent.utils.exception import RemoteToolError +from erniebot_agent.utils.exceptions import RemoteToolError from erniebot_agent.utils.logging import logger @@ -183,7 +183,6 @@ async def send_request(self, tool_arguments: Dict[str, Any]) -> dict: raise RemoteToolError( f"Unsupported content type: {self.tool_view.parameters_content_type}", stage="Executing" ) - if self.tool_view.method == "get": response = requests.get(url, **requests_inputs) # type: ignore elif self.tool_view.method == "post": diff --git a/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py b/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py index d90d04f86..a2bcf1b16 100644 --- a/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py +++ b/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py @@ -2,17 +2,17 @@ import copy import json +import logging import os import tempfile from dataclasses import asdict, dataclass, field from typing import Any, ClassVar, Dict, List, Optional, Type -import erniebot import requests from openapi_spec_validator.readers import read_from_filename from yaml import safe_dump -from erniebot_agent.file_io import get_file_manager +from erniebot_agent.file_io import get_global_file_manager from erniebot_agent.file_io.file_manager import FileManager from erniebot_agent.messages import AIMessage, FunctionCall, HumanMessage, Message from erniebot_agent.tools.remote_tool import RemoteTool, tool_registor @@ -24,9 +24,11 @@ scrub_dict, ) from erniebot_agent.tools.utils import validate_openapi_yaml -from erniebot_agent.utils.exception import RemoteToolError +from erniebot_agent.utils import config_from_environ as C +from erniebot_agent.utils.exceptions import RemoteToolError from erniebot_agent.utils.http import url_file_exists -from erniebot_agent.utils.logging import logger + +logger = logging.getLogger(__name__) @dataclass @@ -125,7 +127,7 @@ def get_tool(self, tool_name: str) -> RemoteTool: TOOL_CLASS = tool_registor.get_tool_class(self.info.title) return TOOL_CLASS( - paths[0], + copy.deepcopy(paths[0]), self.servers[0].url, self.headers, self.info.version, @@ -170,6 +172,9 @@ def from_openapi_dict( info = EndpointInfo(**openapi_dict["info"]) servers = [Endpoint(**server) for server in openapi_dict.get("servers", [])] + if access_token is None: + access_token = C.get_global_access_token() + # components component_schemas = openapi_dict["components"]["schemas"] fields = {} @@ -192,7 +197,7 @@ def from_openapi_dict( ) if file_manager is None: - file_manager = get_file_manager(access_token) + file_manager = get_global_file_manager() return RemoteToolkit( openapi=openapi_dict["openapi"], @@ -217,6 +222,9 @@ def from_openapi_file( if not validate_openapi_yaml(file): raise RemoteToolError(f"invalid openapi yaml file: {file}", stage="Loading") + if access_token is None: + access_token = C.get_global_access_token() + spec_dict, _ = read_from_filename(file) return cls.from_openapi_dict( spec_dict, access_token=access_token, file_manager=file_manager # type: ignore @@ -224,9 +232,6 @@ def from_openapi_file( @classmethod def _get_authorization_headers(cls, access_token: Optional[str]) -> dict: - if access_token is None: - access_token = erniebot.access_token - headers = {"Content-Type": "application/json"} if access_token is None: logger.warning("access_token is NOT provided, this may cause 403 HTTP error..") @@ -244,6 +249,9 @@ def from_aistudio( ) -> RemoteToolkit: from urllib.parse import urlparse + if access_token is None: + access_token = C.get_global_access_token() + aistudio_base_url = os.getenv("AISTUDIO_HUB_BASE_URL", cls._AISTUDIO_HUB_BASE_URL) parsed_url = urlparse(aistudio_base_url) tool_url = parsed_url._replace(netloc=f"tool-{tool_id}.{parsed_url.netloc}").geturl() @@ -265,6 +273,9 @@ def from_url( if version: openapi_yaml_url = openapi_yaml_url + "?version=" + version + if access_token is None: + access_token = C.get_global_access_token() + with tempfile.TemporaryDirectory() as temp_dir: response = requests.get(openapi_yaml_url, headers=cls._get_authorization_headers(access_token)) if response.status_code != 200: @@ -304,6 +315,9 @@ def load_remote_examples_yaml(cls, url: str, access_token: Optional[str] = None) if not url_file_exists(examples_yaml_url, cls._get_authorization_headers(access_token)): return [] + if access_token is None: + access_token = C.get_global_access_token() + examples = [] with tempfile.TemporaryDirectory() as temp_dir: response = requests.get(examples_yaml_url, headers=cls._get_authorization_headers(access_token)) diff --git a/erniebot-agent/src/erniebot_agent/tools/schema.py b/erniebot-agent/src/erniebot_agent/tools/schema.py index 32cdec1ea..aa22bb052 100644 --- a/erniebot-agent/src/erniebot_agent/tools/schema.py +++ b/erniebot-agent/src/erniebot_agent/tools/schema.py @@ -24,7 +24,7 @@ from pydantic.fields import FieldInfo from erniebot_agent.utils.common import create_enum_class -from erniebot_agent.utils.exception import RemoteToolError +from erniebot_agent.utils.exceptions import RemoteToolError INVALID_FIELD_NAME = "__invalid_field_name__" diff --git a/erniebot-agent/src/erniebot_agent/tools/utils.py b/erniebot-agent/src/erniebot_agent/tools/utils.py index d7e062106..4e91a1a5b 100644 --- a/erniebot-agent/src/erniebot_agent/tools/utils.py +++ b/erniebot-agent/src/erniebot_agent/tools/utils.py @@ -18,7 +18,7 @@ get_typing_list_type, ) from erniebot_agent.utils.common import get_file_suffix, is_json_response -from erniebot_agent.utils.exception import RemoteToolError +from erniebot_agent.utils.exceptions import RemoteToolError from erniebot_agent.utils.logging import logger diff --git a/erniebot-agent/src/erniebot_agent/utils/config_from_environ.py b/erniebot-agent/src/erniebot_agent/utils/config_from_environ.py new file mode 100644 index 000000000..83b883fbb --- /dev/null +++ b/erniebot-agent/src/erniebot_agent/utils/config_from_environ.py @@ -0,0 +1,36 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +from typing import Optional + + +def get_global_access_token() -> Optional[str]: + return _get_val_from_env_var("EB_AGENT_ACCESS_TOKEN") + + +def get_global_save_dir() -> Optional[str]: + return _get_val_from_env_var("EB_AGENT_SAVE_DIR") + + +def get_logging_level() -> Optional[str]: + return _get_val_from_env_var("EB_AGENT_LOGGING_LEVEL") + + +def get_logging_file_path() -> Optional[str]: + return _get_val_from_env_var("EB_AGENT_LOGGING_FILE") + + +def _get_val_from_env_var(env_var: str) -> Optional[str]: + return os.getenv(env_var) diff --git a/erniebot-agent/src/erniebot_agent/utils/exception.py b/erniebot-agent/src/erniebot_agent/utils/exceptions.py similarity index 95% rename from erniebot-agent/src/erniebot_agent/utils/exception.py rename to erniebot-agent/src/erniebot_agent/utils/exceptions.py index 3ca029cd2..4e7475927 100644 --- a/erniebot-agent/src/erniebot_agent/utils/exception.py +++ b/erniebot-agent/src/erniebot_agent/utils/exceptions.py @@ -36,3 +36,7 @@ def __init__(self, message: str): def __str__(self): return self.message + + +class ObjectClosedError(Exception): + pass diff --git a/erniebot-agent/src/erniebot_agent/utils/logging.py b/erniebot-agent/src/erniebot_agent/utils/logging.py index 2a4a089f3..a3feb7772 100644 --- a/erniebot-agent/src/erniebot_agent/utils/logging.py +++ b/erniebot-agent/src/erniebot_agent/utils/logging.py @@ -13,12 +13,12 @@ # limitations under the License. import logging -import os import re from typing import Dict, List, Optional, Union from erniebot_agent import messages from erniebot_agent.messages import FunctionMessage, Message +from erniebot_agent.utils import config_from_environ as C from erniebot_agent.utils.json import to_pretty_json from erniebot_agent.utils.output_style import ColoredContent @@ -173,10 +173,8 @@ def setup_logging( Raises: ValueError: If the provided verbosity is not a valid logging level. """ - global logger - if verbosity is None: - verbosity = os.environ.get("EB_LOGGING_LEVEL", None) + verbosity = C.get_logging_level() if verbosity is not None: numeric_level = getattr(logging, verbosity.upper(), None) @@ -192,14 +190,11 @@ def setup_logging( logger.addHandler(console_handler) set_role_color() - if use_file_handler or os.getenv("EB_LOGGING_FILE"): - log_file_path = os.getenv("EB_LOGGING_FILE") - if log_file_path is not None: - file_name = os.path.join(log_file_path, "erniebot-agent.log") - file_handler = logging.FileHandler(file_name) - print(log_file_path) - else: - file_handler = logging.FileHandler("erniebot-agent.log") + log_file_path = C.get_logging_file_path() + if log_file_path is None: + log_file_path = "erniebot-agent.log" + if use_file_handler or log_file_path: + file_handler = logging.FileHandler(log_file_path) file_handler.setFormatter(FileFormatter("%(message)s")) logger.addHandler(file_handler) diff --git a/erniebot-agent/src/erniebot_agent/utils/gradio_mixin.py b/erniebot-agent/src/erniebot_agent/utils/mixins.py similarity index 85% rename from erniebot-agent/src/erniebot_agent/utils/gradio_mixin.py rename to erniebot-agent/src/erniebot_agent/utils/mixins.py index 38d56ace8..8e3d5fedf 100644 --- a/erniebot-agent/src/erniebot_agent/utils/gradio_mixin.py +++ b/erniebot-agent/src/erniebot_agent/utils/mixins.py @@ -1,14 +1,33 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from __future__ import annotations + import base64 import os import tempfile -from typing import Any, List +from typing import TYPE_CHECKING, Any, List, Protocol -from erniebot_agent.file_io.base import File -from erniebot_agent.file_io.file_manager import FileManager -from erniebot_agent.tools.tool_manager import ToolManager from erniebot_agent.utils.common import get_file_type +from erniebot_agent.utils.exceptions import ObjectClosedError from erniebot_agent.utils.html_format import IMAGE_HTML, ITEM_LIST_HTML +if TYPE_CHECKING: + from erniebot_agent.file_io.base import File + from erniebot_agent.file_io.file_manager import FileManager + from erniebot_agent.tools.tool_manager import ToolManager + class GradioMixin: _file_manager: FileManager # make mypy happy @@ -93,7 +112,7 @@ def _clear(): self.reset_memory() return None, None, None, None - async def _upload(file: List[gr.utils.NamedString], history: list): + async def _upload(file, history): nonlocal _uploaded_file_cache for single_file in file: upload_file = await self._file_manager.create_file_from_path(single_file.name) @@ -101,7 +120,7 @@ async def _upload(file: List[gr.utils.NamedString], history: list): history = history + [((single_file.name,), None)] size = len(file) - output_lis = self._file_manager.registry.list_files() + output_lis = self._file_manager.list_registered_files() item = "" for i in range(len(output_lis) - size): item += f'
  • {str(output_lis[i]).strip("<>")}
    • ' @@ -146,7 +165,7 @@ def _messages_to_dicts(messages): ) with gr.Accordion("Files", open=False): - file_lis = self._file_manager.registry.list_files() + file_lis = self._file_manager.list_registered_files() all_files = gr.HTML(value=file_lis, label="All input files") with gr.Accordion("Tools", open=False): attached_tools = self._tool_manager.get_tools() @@ -209,3 +228,16 @@ def _messages_to_dicts(messages): else: allowed_paths = [td] demo.launch(allowed_paths=allowed_paths, **launch_kwargs) + + +class Closeable(Protocol): + @property + def closed(self) -> bool: + ... + + async def close(self) -> None: + ... + + def ensure_not_closed(self) -> None: + if self.closed: + raise ObjectClosedError(f"{repr(self)} is closed.") diff --git a/erniebot-agent/src/erniebot_agent/utils/temp_file.py b/erniebot-agent/src/erniebot_agent/utils/temp_file.py new file mode 100644 index 000000000..ecb98ac43 --- /dev/null +++ b/erniebot-agent/src/erniebot_agent/utils/temp_file.py @@ -0,0 +1,39 @@ +# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import atexit +import logging +import pathlib +import tempfile +from tempfile import TemporaryDirectory +from typing import Any, List + +logger = logging.getLogger(__name__) + +_tracked_temp_dirs: List[TemporaryDirectory] = [] + + +def create_tracked_temp_dir(*args: Any, **kwargs: Any) -> pathlib.Path: + # Borrowed from + # https://github.com/pypa/pipenv/blob/247a14369d300a6980a8dd634d9060bf6f582d2d/pipenv/utils/fileutils.py#L197 + def _cleanup() -> None: + try: + temp_dir.cleanup() + except Exception as e: + logger.warning("Failed to clean up temporary directory: %s", temp_dir.name, exc_info=e) + + temp_dir = tempfile.TemporaryDirectory(*args, **kwargs) + _tracked_temp_dirs.append(temp_dir) + atexit.register(_cleanup) + return pathlib.Path(temp_dir.name) From f1783001a23a66489cda89be29d620e8b8554bba Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Thu, 21 Dec 2023 18:10:12 +0800 Subject: [PATCH 05/63] Update examples --- erniebot-agent/examples/cv_agent/CV_agent.py | 12 ++++++++---- erniebot-agent/examples/plugins/multiple_plugins.py | 6 +++--- erniebot-agent/examples/rpg_game_agent.py | 4 ++-- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/erniebot-agent/examples/cv_agent/CV_agent.py b/erniebot-agent/examples/cv_agent/CV_agent.py index 3eae060f0..6a397da70 100644 --- a/erniebot-agent/examples/cv_agent/CV_agent.py +++ b/erniebot-agent/examples/cv_agent/CV_agent.py @@ -2,7 +2,10 @@ from erniebot_agent.agents.functional_agent import FunctionalAgent from erniebot_agent.chat_models.erniebot import ERNIEBot -from erniebot_agent.file_io import get_file_manager +from erniebot_agent.file_io import ( + configure_global_file_manager, + get_global_file_manager, +) from erniebot_agent.memory.whole_memory import WholeMemory from erniebot_agent.tools import RemoteToolkit @@ -14,14 +17,15 @@ def __init__(self): self.tools = self.toolkit.get_tools() -llm = ERNIEBot(model="ernie-3.5", api_type="aistudio", access_token="") +configure_global_file_manager(access_token="") +llm = ERNIEBot(model="ernie-bot", api_type="aistudio", access_token="") toolkit = CVToolkit() memory = WholeMemory() -file_manager = get_file_manager() -agent = FunctionalAgent(llm=llm, tools=toolkit.tools, memory=memory, file_manager=file_manager) +agent = FunctionalAgent(llm=llm, tools=toolkit.tools, memory=memory) async def run_agent(): + file_manager = get_global_file_manager() seg_file = await file_manager.create_file_from_path(file_path="cityscapes_demo.png", file_type="local") clas_file = await file_manager.create_file_from_path(file_path="class_img.jpg", file_type="local") ocr_file = await file_manager.create_file_from_path(file_path="ch.png", file_type="local") diff --git a/erniebot-agent/examples/plugins/multiple_plugins.py b/erniebot-agent/examples/plugins/multiple_plugins.py index d858410e8..29d628813 100644 --- a/erniebot-agent/examples/plugins/multiple_plugins.py +++ b/erniebot-agent/examples/plugins/multiple_plugins.py @@ -6,7 +6,7 @@ from erniebot_agent.agents.callback.default import get_no_ellipsis_callback from erniebot_agent.agents.functional_agent import FunctionalAgent from erniebot_agent.chat_models.erniebot import ERNIEBot -from erniebot_agent.file_io import get_file_manager +from erniebot_agent.file_io import get_global_file_manager from erniebot_agent.memory.sliding_window_memory import SlidingWindowMemory from erniebot_agent.messages import AIMessage, HumanMessage, Message from erniebot_agent.tools.base import Tool @@ -32,7 +32,7 @@ async def __call__(self, input_file_id: str, repeat_times: int) -> Dict[str, Any if "" in input_file_id: input_file_id = input_file_id.split("")[0] - file_manager = get_file_manager() # Access_token needs to be set here. + file_manager = get_global_file_manager() input_file = file_manager.look_up_file_by_id(input_file_id) if input_file is None: raise RuntimeError("File not found") @@ -109,7 +109,7 @@ def examples(self) -> List[Message]: # TODO(shiyutang): replace this when model is online llm = ERNIEBot(model="ernie-3.5", api_type="custom") memory = SlidingWindowMemory(max_round=1) -file_manager = get_file_manager(access_token="") # Access_token needs to be set here. +file_manager = get_global_file_manager() # plugins = ["ChatFile", "eChart"] plugins: List[str] = [] agent = FunctionalAgent( diff --git a/erniebot-agent/examples/rpg_game_agent.py b/erniebot-agent/examples/rpg_game_agent.py index 0ef00b23e..2c9aecfba 100644 --- a/erniebot-agent/examples/rpg_game_agent.py +++ b/erniebot-agent/examples/rpg_game_agent.py @@ -24,7 +24,7 @@ from erniebot_agent.agents.base import Agent from erniebot_agent.agents.schema import AgentFile, AgentResponse from erniebot_agent.chat_models.erniebot import ERNIEBot -from erniebot_agent.file_io import get_file_manager +from erniebot_agent.file_io import get_global_file_manager from erniebot_agent.file_io.base import File from erniebot_agent.file_io.file_manager import FileManager from erniebot_agent.memory.sliding_window_memory import SlidingWindowMemory @@ -87,7 +87,7 @@ def __init__( tools=tools, system_message=system_message, ) - self.file_manager: FileManager = get_file_manager() + self.file_manager: FileManager = get_global_file_manager() async def handle_tool(self, tool_name: str, tool_args: str) -> str: tool_response = await self._async_run_tool( From 4cd12a1bdebaa4bb944e2991488fe0b7acd0d696 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Thu, 21 Dec 2023 18:10:19 +0800 Subject: [PATCH 06/63] Update tests --- .../tests/integration_tests/apihub/base.py | 9 +- .../apihub/test_doc_analysis.py | 10 +- .../apihub/test_handwriting.py | 10 +- .../apihub/test_img_transform.py | 10 +- .../integration_tests/apihub/test_ocr.py | 9 +- .../apihub/test_pp_shituv2.py | 4 +- .../agents/callback/test_callback_manager.py | 31 ++- .../agents/test_agent_response_annotations.py | 16 +- .../agents/test_functional_agent.py | 31 ++- .../tests/unit_tests/file/test_local_file.py | 17 ++ .../mocks/mock_remote_file_client_server.py | 187 ++++++++++++++++++ .../unit_tests/tools/test_file_in_tool.py | 43 ++-- 12 files changed, 299 insertions(+), 78 deletions(-) create mode 100644 erniebot-agent/tests/unit_tests/file/test_local_file.py create mode 100644 erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py diff --git a/erniebot-agent/tests/integration_tests/apihub/base.py b/erniebot-agent/tests/integration_tests/apihub/base.py index a7384efb8..1a8c92812 100644 --- a/erniebot-agent/tests/integration_tests/apihub/base.py +++ b/erniebot-agent/tests/integration_tests/apihub/base.py @@ -10,19 +10,20 @@ from erniebot_agent.agents.functional_agent import FunctionalAgent from erniebot_agent.chat_models import ERNIEBot -from erniebot_agent.file_io import get_file_manager +from erniebot_agent.file_io.file_manager import FileManager from erniebot_agent.memory import WholeMemory from erniebot_agent.tools import RemoteToolkit from erniebot_agent.tools.tool_manager import ToolManager class RemoteToolTesting(unittest.IsolatedAsyncioTestCase): - def setUp(self) -> None: + async def asyncSetUp(self) -> None: self.temp_dir = tempfile.mkdtemp() - self.file_manager = get_file_manager() + self.file_manager = FileManager() - def tearDown(self) -> None: + async def asyncTearDown(self) -> None: shutil.rmtree(self.temp_dir) + await self.file_manager.close() def download_file(self, url, file_name: Optional[str] = None): image_response = requests.get(url) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_doc_analysis.py b/erniebot-agent/tests/integration_tests/apihub/test_doc_analysis.py index 7b38f282c..b99342a3b 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_doc_analysis.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_doc_analysis.py @@ -1,7 +1,5 @@ from __future__ import annotations -import asyncio - import pytest from erniebot_agent.tools.remote_toolkit import RemoteToolkit @@ -10,11 +8,9 @@ class TestRemoteTool(RemoteToolTesting): - def setUp(self) -> None: - super().setUp() - self.file = asyncio.run( - self.file_manager.create_file_from_path(self.download_fixture_file("城市管理执法办法.pdf")) - ) + async def asyncSetUp(self) -> None: + await super().asyncSetUp() + self.file = await self.file_manager.create_file_from_path(self.download_fixture_file("城市管理执法办法.pdf")) @pytest.mark.asyncio async def test_doc_analysis(self): diff --git a/erniebot-agent/tests/integration_tests/apihub/test_handwriting.py b/erniebot-agent/tests/integration_tests/apihub/test_handwriting.py index 58eba1471..3d3d1c021 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_handwriting.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_handwriting.py @@ -1,7 +1,5 @@ from __future__ import annotations -import asyncio - import pytest from erniebot_agent.tools.remote_toolkit import RemoteToolkit @@ -10,10 +8,10 @@ class TestRemoteTool(RemoteToolTesting): - def setUp(self) -> None: - super().setUp() - self.file = asyncio.run( - self.file_manager.create_file_from_path(self.download_fixture_file("shouxiezi.png")) + async def asyncSetUp(self) -> None: + await super().asyncSetUp() + self.file = await self.file_manager.create_file_from_path( + self.download_fixture_file("shouxiezi.png") ) @pytest.mark.asyncio diff --git a/erniebot-agent/tests/integration_tests/apihub/test_img_transform.py b/erniebot-agent/tests/integration_tests/apihub/test_img_transform.py index 4115d36ca..7482cacb4 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_img_transform.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_img_transform.py @@ -1,7 +1,5 @@ from __future__ import annotations -import asyncio - import pytest from erniebot_agent.tools.remote_toolkit import RemoteToolkit @@ -10,11 +8,9 @@ class TestRemoteTool(RemoteToolTesting): - def setUp(self) -> None: - super().setUp() - self.file = asyncio.run( - self.file_manager.create_file_from_path(self.download_fixture_file("trans.png")) - ) + async def asyncSetUp(self) -> None: + await super().asyncSetUp() + self.file = await self.file_manager.create_file_from_path(self.download_fixture_file("trans.png")) @pytest.mark.asyncio async def test_img_style_trans(self): diff --git a/erniebot-agent/tests/integration_tests/apihub/test_ocr.py b/erniebot-agent/tests/integration_tests/apihub/test_ocr.py index 4e1c199cb..3199a1164 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_ocr.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_ocr.py @@ -1,6 +1,5 @@ from __future__ import annotations -import asyncio import json import pytest @@ -11,10 +10,10 @@ class TestRemoteTool(RemoteToolTesting): - def setUp(self) -> None: - super().setUp() - self.file = asyncio.run( - self.file_manager.create_file_from_path(self.download_fixture_file("ocr_table.png")) + async def asyncSetUp(self) -> None: + await super().asyncSetUp() + self.file = await self.file_manager.create_file_from_path( + self.download_fixture_file("ocr_table.png") ) @pytest.mark.asyncio diff --git a/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py b/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py index 70dbc021e..bdf8262dd 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py @@ -2,7 +2,7 @@ import pytest -from erniebot_agent.file_io import get_file_manager +from erniebot_agent.file_io import get_global_file_manager from erniebot_agent.tools import RemoteToolkit from .base import RemoteToolTesting @@ -17,7 +17,7 @@ async def test_pp_shituv2(self): agent = self.get_agent(toolkit) - file_manager = get_file_manager() + file_manager = get_global_file_manager() file_path = self.download_file( "https://paddlenlp.bj.bcebos.com/ebagent/ci/fixtures/remote-tools/pp_shituv2_input_img.png" ) diff --git a/erniebot-agent/tests/unit_tests/agents/callback/test_callback_manager.py b/erniebot-agent/tests/unit_tests/agents/callback/test_callback_manager.py index 7f3b30fb0..03f6a715f 100644 --- a/erniebot-agent/tests/unit_tests/agents/callback/test_callback_manager.py +++ b/erniebot-agent/tests/unit_tests/agents/callback/test_callback_manager.py @@ -13,7 +13,7 @@ @pytest.mark.asyncio async def test_callback_manager_hit(): - def _assert_num_calls(handler): + def _assert_all_counts(handler): assert handler.run_starts == 1 assert handler.llm_starts == 1 assert handler.llm_ends == 1 @@ -27,27 +27,40 @@ def _assert_num_calls(handler): llm = mock.Mock(spec=ChatModel) tool = mock.Mock(spec=Tool) - handler1 = CountingCallbackHandler() - handler2 = CountingCallbackHandler() - callback_manager = CallbackManager(handlers=[handler1, handler2]) + handler = CountingCallbackHandler() + callback_manager = CallbackManager(handlers=[handler]) await callback_manager.on_run_start(agent, "") + assert handler.run_starts == 1 + await callback_manager.on_llm_start(agent, llm, []) + assert handler.llm_starts == 1 + await callback_manager.on_llm_end( agent, llm, AIMessage(content="", function_call=None, token_usage={"prompt_tokens": 0, "completion_tokens": 0}), ) + assert handler.llm_ends == 1 + await callback_manager.on_llm_error(agent, llm, Exception()) + assert handler.llm_errors == 1 + await callback_manager.on_tool_start(agent, tool, "{}") + assert handler.tool_starts == 1 + await callback_manager.on_tool_end(agent, tool, "{}") + assert handler.tool_ends == 1 + await callback_manager.on_tool_error(agent, tool, Exception()) + assert handler.tool_errors == 1 + await callback_manager.on_run_end( agent, AgentResponse(text="", chat_history=[], actions=[], files=[], status="FINISHED") ) + assert handler.run_ends == 1 - _assert_num_calls(handler1) - _assert_num_calls(handler2) + _assert_all_counts(handler) @pytest.mark.asyncio @@ -55,14 +68,20 @@ async def test_callback_manager_add_remove_handlers(): handler1 = CountingCallbackHandler() handler2 = CountingCallbackHandler() callback_manager = CallbackManager(handlers=[handler1]) + assert len(callback_manager.handlers) == 1 + with pytest.raises(RuntimeError): callback_manager.add_handler(handler1) + callback_manager.remove_handler(handler1) assert len(callback_manager.handlers) == 0 + callback_manager.add_handler(handler1) assert len(callback_manager.handlers) == 1 + callback_manager.add_handler(handler2) assert len(callback_manager.handlers) == 2 + callback_manager.remove_all_handlers() assert len(callback_manager.handlers) == 0 diff --git a/erniebot-agent/tests/unit_tests/agents/test_agent_response_annotations.py b/erniebot-agent/tests/unit_tests/agents/test_agent_response_annotations.py index 6253327ae..eced30299 100644 --- a/erniebot-agent/tests/unit_tests/agents/test_agent_response_annotations.py +++ b/erniebot-agent/tests/unit_tests/agents/test_agent_response_annotations.py @@ -1,17 +1,19 @@ -import asyncio import unittest from typing import List, Literal from erniebot_agent.agents.schema import AgentFile, AgentResponse -from erniebot_agent.file_io import get_file_manager +from erniebot_agent.file_io.file_manager import FileManager -class TestAgentResponseAnnotations(unittest.TestCase): - def setUp(self): +class TestAgentResponseAnnotations(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): self.test = "" - self.file_manager = get_file_manager() - self.file1 = asyncio.run(self.file_manager.create_file_from_bytes(b"test1", "test1.txt")) - self.file2 = asyncio.run(self.file_manager.create_file_from_bytes(b"test2", "test2.txt")) + self.file_manager = FileManager() + self.file1 = await self.file_manager.create_file_from_bytes(b"test1", "test1.txt") + self.file2 = await self.file_manager.create_file_from_bytes(b"test2", "test2.txt") + + async def asyncTearDown(self): + await self.file_manager.close() def test_agent_response_onefile_oneagentfile(self): agent_file = AgentFile(file=self.file1, type="input", used_by="") diff --git a/erniebot-agent/tests/unit_tests/agents/test_functional_agent.py b/erniebot-agent/tests/unit_tests/agents/test_functional_agent.py index d3a886aa4..b6ef7e4f6 100644 --- a/erniebot-agent/tests/unit_tests/agents/test_functional_agent.py +++ b/erniebot-agent/tests/unit_tests/agents/test_functional_agent.py @@ -97,17 +97,22 @@ async def test_functional_agent_load_unload_tools(identity_tool, no_input_no_out @pytest.mark.asyncio -async def test_functional_agent_run_llm(identity_tool): +async def test_functional_agent_run_llm_return_text(): output_message = AIMessage("Hello!", function_call=None) agent = FunctionalAgent( llm=FakeChatModelWithPresetResponses(responses=[output_message]), tools=[], memory=FakeMemory(), ) + llm_response = await agent._async_run_llm(messages=[HumanMessage("Hello, world!")]) + assert isinstance(llm_response.message, AIMessage) assert llm_response.message == output_message + +@pytest.mark.asyncio +async def test_functional_agent_run_llm_return_function_call(identity_tool): output_message = AIMessage( "", function_call=FunctionCall( @@ -119,7 +124,9 @@ async def test_functional_agent_run_llm(identity_tool): tools=[identity_tool], memory=FakeMemory(), ) + llm_response = await agent._async_run_llm(messages=[HumanMessage("Hello, world!")]) + assert isinstance(llm_response.message, AIMessage) assert llm_response.message == output_message @@ -161,24 +168,6 @@ async def test_functional_agent_memory(identity_tool): AIMessage("", function_call=function_call), AIMessage("", function_call=function_call), AIMessage(output_text, function_call=None), - ] - ) - agent = FunctionalAgent( - llm=llm, - tools=[identity_tool], - memory=FakeMemory(), - ) - await agent.async_run(input_text) - messages_in_memory = agent.memory.get_messages() - assert len(messages_in_memory) == 2 - assert isinstance(messages_in_memory[0], HumanMessage) - assert messages_in_memory[0].content == input_text - assert isinstance(messages_in_memory[1], AIMessage) - assert messages_in_memory[1].content == output_text - - llm = FakeChatModelWithPresetResponses( - responses=[ - AIMessage(output_text, function_call=None), AIMessage(output_text, function_call=None), AIMessage("This message should not be remembered.", function_call=None), AIMessage("This message should not be remembered, either.", function_call=None), @@ -189,6 +178,7 @@ async def test_functional_agent_memory(identity_tool): tools=[identity_tool], memory=FakeMemory(), ) + await agent.async_run(input_text) messages_in_memory = agent.memory.get_messages() assert len(messages_in_memory) == 2 @@ -196,6 +186,7 @@ async def test_functional_agent_memory(identity_tool): assert messages_in_memory[0].content == input_text assert isinstance(messages_in_memory[1], AIMessage) assert messages_in_memory[1].content == output_text + await agent.async_run(input_text) assert len(agent.memory.get_messages()) == 2 + 2 agent.reset_memory() @@ -223,5 +214,7 @@ async def test_functional_agent_max_steps(identity_tool): memory=FakeMemory(), max_steps=2, ) + response = await agent.async_run("Run!") + assert response.status == "STOPPED" diff --git a/erniebot-agent/tests/unit_tests/file/test_local_file.py b/erniebot-agent/tests/unit_tests/file/test_local_file.py new file mode 100644 index 000000000..b06052ba3 --- /dev/null +++ b/erniebot-agent/tests/unit_tests/file/test_local_file.py @@ -0,0 +1,17 @@ +import pathlib +import tempfile + +import erniebot_agent.file_io.protocol as protocol +from erniebot_agent.file_io.local_file import LocalFile, create_local_file_from_path + + +def test_create_local_file_from_path(): + with tempfile.TemporaryDirectory() as td: + file_path = pathlib.Path(td) / "temp_file" + file_path.touch() + file_purpose = "assistants" + + file = create_local_file_from_path(file_path, file_purpose, {}) + + assert isinstance(file, LocalFile) + assert protocol.is_local_file_id(file.id) diff --git a/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py b/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py new file mode 100644 index 000000000..188173f7a --- /dev/null +++ b/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py @@ -0,0 +1,187 @@ +import contextlib +import re +import uuid + +import erniebot_agent.file_io.protocol as protocol +from erniebot_agent.file_io.remote_file import RemoteFile, RemoteFileClient + + +class FakeRemoteFileProtocol(object): + _UUID_PATTERN = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" + _FILE_ID_PREFIX = r"file-fake-remote-" + _FILE_ID_PATTERN = _FILE_ID_PREFIX + _UUID_PATTERN + + _followed = False + + @classmethod + def is_remote_file_id(cls, str_): + return re.fullmatch(cls._FILE_ID_PATTERN, str_) is not None + + @classmethod + def extract_remote_file_ids(cls, str_): + return re.findall(cls._FILE_ID_PATTERN, str_) + + @classmethod + def generate_remote_file_id(cls): + return cls._FILE_ID_PREFIX + str(uuid.uuid4()) + + get_timestamp = protocol.get_timestamp + + @classmethod + @contextlib.contextmanager + def follow(cls, old_protocol=None): + if not cls._followed: + names_methods_to_monkey_patch = ( + "is_remote_file_id", + "extract_remote_file_ids", + "get_timestamp", + ) + + if old_protocol is None: + old_protocol = protocol + + _old_methods = {} + + for method_name in names_methods_to_monkey_patch: + old_method = getattr(old_protocol, method_name) + new_method = getattr(cls, method_name) + _old_methods[method_name] = old_method + setattr(old_protocol, method_name, new_method) + + cls._followed = True + + yield + + for method_name in names_methods_to_monkey_patch: + old_method = _old_methods[method_name] + setattr(old_protocol, method_name, old_method) + + cls._followed = False + + else: + yield + + +class FakeRemoteFileClient(RemoteFileClient): + _protocol = FakeRemoteFileProtocol + + def __init__(self, server): + super().__init__() + if server.protocol is not self.protocol: + raise ValueError("Server and client do not share the same protocol.") + self._server = server + + @property + def protocol(self): + return self._protocol + + @property + def server(self): + if not self._server.started: + raise RuntimeError("Server is not running.") + return self._server + + async def upload_file(self, file_path, file_purpose, file_metadata): + result = await self.server.upload_file(file_path, file_purpose, file_metadata) + return self._create_file_obj_from_dict(result) + + async def retrieve_file(self, file_id): + result = await self.server.retrieve_file(file_id) + return self._create_file_obj_from_dict(result) + + async def retrieve_file_contents(self, file_id): + return await self.server.retrieve_file_contents(file_id) + + async def list_files(self): + result = await self.server.list_files() + files = [] + for item in result: + file = self._create_file_obj_from_dict(item) + files.append(file) + return files + + async def delete_file(self, file_id) -> None: + await self.server.delete_file(file_id) + + async def create_temporary_url(self, file_id, expire_after): + raise RuntimeError("Method not supported") + + def _create_file_obj_from_dict(self, dict_): + with self._protocol.follow(): + return RemoteFile( + id=dict_["id"], + filename=dict_["filename"], + byte_size=dict_["byte_size"], + created_at=dict_["created_at"], + purpose=dict_["purpose"], + metadata=dict_["metadata"], + client=self, + ) + + +class FakeRemoteFileServer(object): + _protocol = FakeRemoteFileProtocol + + def __init__(self): + super().__init__() + self._storage = None + + @property + def protocol(self): + return self._protocol + + @property + def storage(self): + return self._storage + + @property + def started(self): + return self._storage is not None + + async def upload_file(self, file_path, file_purpose, file_metadata): + id_ = self._protocol.generate_remote_file_id() + filename = file_path.name + byte_size = file_path.stat().st_size + created_at = self._protocol.get_timestamp() + with file_path.open("rb") as f: + contents = f.read() + file = dict( + id=id_, + filename=filename, + byte_size=byte_size, + created_at=created_at, + purpose=file_purpose, + metadata=file_metadata, + contents=contents, + ) + self._storage[id_] = file + return file + + async def retrieve_file(self, file_id): + try: + return self._storage[file_id] + except KeyError as e: + raise RuntimeError("File not found") from e + + async def retrieve_file_contents(self, file_id): + try: + file = self._storage[file_id] + except KeyError as e: + raise RuntimeError("File not found") from e + else: + return file["contents"] + + async def list_files(self): + return list(self._storage.values()) + + async def delete_file(self, file_id) -> None: + try: + return self._storage[file_id] + except KeyError as e: + raise RuntimeError("File not found") from e + + @contextlib.contextmanager + def start(self): + self._storage = {} + yield self + self._storage = None diff --git a/erniebot-agent/tests/unit_tests/tools/test_file_in_tool.py b/erniebot-agent/tests/unit_tests/tools/test_file_in_tool.py index 0e81e9163..80fa7314b 100644 --- a/erniebot-agent/tests/unit_tests/tools/test_file_in_tool.py +++ b/erniebot-agent/tests/unit_tests/tools/test_file_in_tool.py @@ -14,7 +14,6 @@ from __future__ import annotations -import asyncio import base64 import json import os @@ -110,7 +109,13 @@ def is_port_in_use(port): return True -class TestToolWithFile(unittest.TestCase): +class TestToolWithFile(unittest.IsolatedAsyncioTestCase): + async def asyncSetUp(self): + self.file_manager = FileManager() + + async def asyncTearDown(self): + await self.file_manager.close() + def avaliable_free_port(self, exclude=None): exclude = exclude or [] for port in range(8000, 9000): @@ -145,7 +150,7 @@ def wait_until_server_is_ready(self): print("waiting for server ...") time.sleep(1) - def test_plugin_schema(self): + async def test_plugin_schema(self): self.wait_until_server_is_ready() with tempfile.TemporaryDirectory() as tempdir: openapi_file = os.path.join(tempdir, "openapi.yaml") @@ -153,25 +158,27 @@ def test_plugin_schema(self): with open(openapi_file, "w", encoding="utf-8") as f: f.write(content) - toolkit = RemoteToolkit.from_openapi_file(openapi_file) + toolkit = RemoteToolkit.from_openapi_file(openapi_file, file_manager=self.file_manager) tool = toolkit.get_tool("getFile") # tool.tool_name should have `tool_name_prefix`` prepended self.assertEqual(tool.tool_name, "TestRemoteTool/v1/getFile") - file_manager = FileManager() - input_file = asyncio.run(file_manager.create_file_from_path(self.file_path)) - result = asyncio.run(tool(file=input_file.id)) + input_file = await self.file_manager.create_file_from_path(self.file_path) + result = await tool(file=input_file.id) self.assertIn("response_file", result) file_id = result["response_file"] - file = file_manager.look_up_file_by_id(file_id=file_id) - content = asyncio.run(file.read_contents()) + file = self.file_manager.look_up_file_by_id(file_id=file_id) + content = await file.read_contents() self.assertEqual(content.decode("utf-8"), self.content) class TestPlainJsonFileParser(unittest.IsolatedAsyncioTestCase): - def setUp(self) -> None: + async def asyncSetUp(self): self.file_manager = FileManager() + async def asyncTearDown(self): + await self.file_manager.close() + def create_fake_response(self, body: dict): the_response = Response() the_response.code = "expired" @@ -224,7 +231,7 @@ async def test_plain_file(self): file_path = os.path.join(temp_dir, "openapi.yaml") with open(file_path, "w+", encoding="utf-8") as f: f.write(yaml_content) - toolkit = RemoteToolkit.from_openapi_file(file_path) + toolkit = RemoteToolkit.from_openapi_file(file_path, file_manager=self.file_manager) response = self.create_fake_response(body) tool = toolkit.get_tools()[-1] @@ -241,9 +248,12 @@ async def test_plain_file(self): class TestJsonNestFileParser(unittest.IsolatedAsyncioTestCase): - def setUp(self) -> None: + async def asyncSetUp(self): self.file_manager = FileManager() + async def asyncTearDown(self): + await self.file_manager.close() + def create_fake_response(self, body: dict): the_response = Response() the_response.code = "expired" @@ -300,7 +310,7 @@ async def test_plain_file(self): file_path = os.path.join(temp_dir, "openapi.yaml") with open(file_path, "w+", encoding="utf-8") as f: f.write(yaml_content) - toolkit = RemoteToolkit.from_openapi_file(file_path) + toolkit = RemoteToolkit.from_openapi_file(file_path, file_manager=self.file_manager) response = self.create_fake_response(body) tool = toolkit.get_tools()[-1] @@ -319,9 +329,12 @@ async def test_plain_file(self): class TestJsonNestListFileParser(unittest.IsolatedAsyncioTestCase): - def setUp(self) -> None: + async def asyncSetUp(self): self.file_manager = FileManager() + async def asyncTearDown(self): + await self.file_manager.close() + def create_fake_response(self, body: dict): the_response = Response() the_response.code = "expired" @@ -381,7 +394,7 @@ async def test_plain_file(self): file_path = os.path.join(temp_dir, "openapi.yaml") with open(file_path, "w+", encoding="utf-8") as f: f.write(yaml_content) - toolkit = RemoteToolkit.from_openapi_file(file_path) + toolkit = RemoteToolkit.from_openapi_file(file_path, file_manager=self.file_manager) response = self.create_fake_response(body) tool = toolkit.get_tools()[-1] From 23287ad84aa50d238bc4a6621ac1cb5db99b5fa3 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Thu, 21 Dec 2023 19:20:33 +0800 Subject: [PATCH 07/63] Fix exceptions --- .../erniebot_agent/agents/callback/callback_manager.py | 4 ++-- .../src/erniebot_agent/file_io/file_registry.py | 4 ++-- .../src/erniebot_agent/tools/tool_manager.py | 6 +++--- .../agents/callback/test_callback_manager.py | 2 +- .../tests/unit_tests/agents/test_functional_agent.py | 6 +++--- .../mocks/mock_remote_file_client_server.py | 10 +++++++--- erniebot/src/erniebot/cli.py | 2 +- erniebot/src/erniebot/resources/resource.py | 10 +++++----- 8 files changed, 24 insertions(+), 20 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/agents/callback/callback_manager.py b/erniebot-agent/src/erniebot_agent/agents/callback/callback_manager.py index 8bf6a9918..17d9949bc 100644 --- a/erniebot-agent/src/erniebot_agent/agents/callback/callback_manager.py +++ b/erniebot-agent/src/erniebot_agent/agents/callback/callback_manager.py @@ -40,14 +40,14 @@ def handlers(self) -> List[CallbackHandler]: def add_handler(self, handler: CallbackHandler): if handler in self._handlers: - raise RuntimeError(f"The callback handler {handler} is already registered.") + raise ValueError(f"The callback handler {handler} is already registered.") self._handlers.append(handler) def remove_handler(self, handler): try: self._handlers.remove(handler) except ValueError as e: - raise RuntimeError(f"The callback handler {handler} is not registered.") from e + raise ValueError(f"The callback handler {handler} is not registered.") from e def set_handlers(self, handlers: List[CallbackHandler]): self._handlers = [] diff --git a/erniebot-agent/src/erniebot_agent/file_io/file_registry.py b/erniebot-agent/src/erniebot_agent/file_io/file_registry.py index 607562db8..79986b572 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/file_registry.py +++ b/erniebot-agent/src/erniebot_agent/file_io/file_registry.py @@ -41,7 +41,7 @@ def register_file(self, file: File, *, allow_overwrite: bool = False, check_type file_id = file.id if file_id in self._id_to_file: if not allow_overwrite: - raise RuntimeError(f"File with ID {repr(file_id)} is already registered.") + raise ValueError(f"File with ID {repr(file_id)} is already registered.") else: if check_type and type(file) is not type(self._id_to_file[file_id]): # noqa: E721 raise RuntimeError("Cannot register a file with a different type.") @@ -50,7 +50,7 @@ def register_file(self, file: File, *, allow_overwrite: bool = False, check_type def unregister_file(self, file: File) -> None: file_id = file.id if file_id not in self._id_to_file: - raise RuntimeError(f"File with ID {repr(file_id)} is not registered.") + raise ValueError(f"File with ID {repr(file_id)} is not registered.") self._id_to_file.pop(file_id) def look_up_file(self, file_id: str) -> Optional[File]: diff --git a/erniebot-agent/src/erniebot_agent/tools/tool_manager.py b/erniebot-agent/src/erniebot_agent/tools/tool_manager.py index 35ed5a7b3..4948545a1 100644 --- a/erniebot-agent/src/erniebot_agent/tools/tool_manager.py +++ b/erniebot-agent/src/erniebot_agent/tools/tool_manager.py @@ -38,20 +38,20 @@ def __getitem__(self, tool_name: str) -> BaseTool: def add_tool(self, tool: BaseTool) -> None: tool_name = tool.tool_name if tool_name in self._tools: - raise RuntimeError(f"Name {repr(tool_name)} is already registered.") + raise ValueError(f"Name {repr(tool_name)} is already registered.") self._tools[tool_name] = tool def remove_tool(self, tool: BaseTool) -> None: tool_name = tool.tool_name if tool_name not in self._tools: - raise RuntimeError(f"Name {repr(tool_name)} is not registered.") + raise ValueError(f"Name {repr(tool_name)} is not registered.") if self._tools[tool_name] is not tool: raise RuntimeError(f"The tool with the registered name {repr(tool_name)} is not the given tool.") self._tools.pop(tool_name) def get_tool(self, tool_name: str) -> BaseTool: if tool_name not in self._tools: - raise RuntimeError(f"Name {repr(tool_name)} is not registered.") + raise ValueError(f"Name {repr(tool_name)} is not registered.") return self._tools[tool_name] def get_tools(self) -> List[BaseTool]: diff --git a/erniebot-agent/tests/unit_tests/agents/callback/test_callback_manager.py b/erniebot-agent/tests/unit_tests/agents/callback/test_callback_manager.py index 03f6a715f..e291466aa 100644 --- a/erniebot-agent/tests/unit_tests/agents/callback/test_callback_manager.py +++ b/erniebot-agent/tests/unit_tests/agents/callback/test_callback_manager.py @@ -71,7 +71,7 @@ async def test_callback_manager_add_remove_handlers(): assert len(callback_manager.handlers) == 1 - with pytest.raises(RuntimeError): + with pytest.raises(ValueError): callback_manager.add_handler(handler1) callback_manager.remove_handler(handler1) diff --git a/erniebot-agent/tests/unit_tests/agents/test_functional_agent.py b/erniebot-agent/tests/unit_tests/agents/test_functional_agent.py index b6ef7e4f6..53d0c21a8 100644 --- a/erniebot-agent/tests/unit_tests/agents/test_functional_agent.py +++ b/erniebot-agent/tests/unit_tests/agents/test_functional_agent.py @@ -88,11 +88,11 @@ async def test_functional_agent_load_unload_tools(identity_tool, no_input_no_out ) agent.load_tool(tool2) - with pytest.raises(RuntimeError): + with pytest.raises(ValueError): agent.load_tool(tool1) agent.unload_tool(tool1) - with pytest.raises(RuntimeError): + with pytest.raises(ValueError): agent.unload_tool(tool1) @@ -148,7 +148,7 @@ async def test_functional_agent_run_tool(identity_tool, no_input_no_output_tool) assert json.loads(tool_response.json) == {} tool_input = {} - with pytest.raises(RuntimeError): + with pytest.raises(ValueError): await agent._async_run_tool("some_tool_name_that_does_not_exist", json.dumps(tool_input)) diff --git a/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py b/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py index 188173f7a..9731127c7 100644 --- a/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py +++ b/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py @@ -161,13 +161,13 @@ async def retrieve_file(self, file_id): try: return self._storage[file_id] except KeyError as e: - raise RuntimeError("File not found") from e + raise ServerError("File not found") from e async def retrieve_file_contents(self, file_id): try: file = self._storage[file_id] except KeyError as e: - raise RuntimeError("File not found") from e + raise ServerError("File not found") from e else: return file["contents"] @@ -178,10 +178,14 @@ async def delete_file(self, file_id) -> None: try: return self._storage[file_id] except KeyError as e: - raise RuntimeError("File not found") from e + raise ServerError("File not found") from e @contextlib.contextmanager def start(self): self._storage = {} yield self self._storage = None + + +class ServerError(Exception): + pass diff --git a/erniebot/src/erniebot/cli.py b/erniebot/src/erniebot/cli.py index dd5517962..c50c882c3 100644 --- a/erniebot/src/erniebot/cli.py +++ b/erniebot/src/erniebot/cli.py @@ -121,7 +121,7 @@ def _find_method(method_name): raise AttributeError(f"{cls.__name__}.{method_name} is not found.") method = getattr(cls, method_name) if not callable(method): - raise TypeError(f"{cls.__name__}.{method_name} is not callable.") + raise RuntimeError(f"{cls.__name__}.{method_name} is not callable.") return method cls.add_resource_arguments(parser) diff --git a/erniebot/src/erniebot/resources/resource.py b/erniebot/src/erniebot/resources/resource.py index 18569ca47..1a2422109 100644 --- a/erniebot/src/erniebot/resources/resource.py +++ b/erniebot/src/erniebot/resources/resource.py @@ -368,12 +368,12 @@ def _request( ) if stream: if not isinstance(resp, Iterator): - raise TypeError("Expected an iterator of response objects") + raise RuntimeError("Expected an iterator of response objects") else: return resp else: if not isinstance(resp, EBResponse): - raise TypeError("Expected a response object") + raise RuntimeError("Expected a response object") else: return resp @@ -438,12 +438,12 @@ async def _arequest( ) if stream: if not isinstance(resp, AsyncIterator): - raise TypeError("Expected an iterator of response objects") + raise RuntimeError("Expected an iterator of response objects") else: return resp else: if not isinstance(resp, EBResponse): - raise TypeError("Expected a response object") + raise RuntimeError("Expected a response object") else: return resp @@ -451,7 +451,7 @@ def _create_config_dict(self, overrides: Any) -> ConfigDictType: cfg_dict = GlobalConfig().create_dict(**overrides) api_type_str = cfg_dict["api_type"] if not isinstance(api_type_str, str): - raise TypeError + raise RuntimeError("Expected a string") api_type = convert_str_to_api_type(api_type_str) cfg_dict["api_type"] = api_type return cfg_dict From 4807109166123cf526699e3303f4ba7c4e86ec79 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Thu, 21 Dec 2023 19:25:42 +0800 Subject: [PATCH 08/63] Fix error info --- .../src/erniebot_agent/file_io/global_file_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py b/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py index caeafb935..5296bc95a 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py @@ -37,7 +37,9 @@ def configure_global_file_manager( ) -> None: global _global_file_manager if _global_file_manager is not None: - raise RuntimeError("The global file manager can only be configured once.") + raise RuntimeError( + "The global file manager can only be configured once before calling `get_global_file_manager`." + ) _global_file_manager = _create_default_file_manager(access_token=access_token, save_dir=save_dir, **opts) From bc8bbd86ee17225df6fae0e44adce89493b31f6e Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Thu, 21 Dec 2023 19:29:29 +0800 Subject: [PATCH 09/63] Remove use of environment variables in integration tests --- .../integration_tests/chat_models/test_chat_model.py | 8 ++------ .../tests/integration_tests/tools/test_calculator.py | 4 ---- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/erniebot-agent/tests/integration_tests/chat_models/test_chat_model.py b/erniebot-agent/tests/integration_tests/chat_models/test_chat_model.py index 22bf144da..8c8f5ca4e 100644 --- a/erniebot-agent/tests/integration_tests/chat_models/test_chat_model.py +++ b/erniebot-agent/tests/integration_tests/chat_models/test_chat_model.py @@ -10,9 +10,7 @@ class TestChatModel(unittest.IsolatedAsyncioTestCase): @pytest.mark.asyncio async def test_chat(self): - eb = ERNIEBot( - model="ernie-turbo", api_type="aistudio", access_token=os.environ["AISTUDIO_ACCESS_TOKEN"] - ) + eb = ERNIEBot(model="ernie-turbo", api_type="aistudio") messages = [ HumanMessage(content="你好！"), ] @@ -65,9 +63,7 @@ async def test_function_call(self): } ] # use ernie-3.5 here since ernie-turbo doesn't support function call - eb = ERNIEBot( - model="ernie-3.5", api_type="aistudio", access_token=os.environ["AISTUDIO_ACCESS_TOKEN"] - ) + eb = ERNIEBot(model="ernie-3.5", api_type="aistudio") messages = [ HumanMessage(content="深圳市今天的气温是多少摄氏度？"), ] diff --git a/erniebot-agent/tests/integration_tests/tools/test_calculator.py b/erniebot-agent/tests/integration_tests/tools/test_calculator.py index 853545f02..16adc77ef 100644 --- a/erniebot-agent/tests/integration_tests/tools/test_calculator.py +++ b/erniebot-agent/tests/integration_tests/tools/test_calculator.py @@ -2,16 +2,12 @@ import asyncio import json -import os import unittest import erniebot from erniebot_agent.tools.calculator_tool import CalculatorTool -erniebot.api_type = "aistudio" -erniebot.access_token = os.environ["AISTUDIO_ACCESS_TOKEN"] - class TestCalculator(unittest.TestCase): def setUp(self) -> None: From b2803333e13348ec3141d671f8ff6b8e7936d5ec Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Thu, 21 Dec 2023 20:55:59 +0800 Subject: [PATCH 10/63] Fix protocol --- .../src/erniebot_agent/file_io/protocol.py | 11 ++- .../chat_models/test_chat_model.py | 1 - .../mocks/mock_remote_file_client_server.py | 98 +++---------------- 3 files changed, 22 insertions(+), 88 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/file_io/protocol.py b/erniebot-agent/src/erniebot_agent/file_io/protocol.py index f636dfe0f..6caad1424 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/protocol.py +++ b/erniebot-agent/src/erniebot_agent/file_io/protocol.py @@ -14,7 +14,7 @@ import datetime import re -from typing import List, Literal +from typing import Generator, List, Literal from typing_extensions import TypeAlias @@ -23,7 +23,8 @@ _LOCAL_FILE_ID_PREFIX = "file-local-" _UUID_PATTERN = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" _LOCAL_FILE_ID_PATTERN = _LOCAL_FILE_ID_PREFIX + _UUID_PATTERN -_REMOTE_FILE_ID_PATTERN = r"file-[0-9]{15}" +_REMOTE_FILE_ID_PREFIX = "file-" +_REMOTE_FILE_ID_PATTERN = _REMOTE_FILE_ID_PREFIX + r"[0-9]{15}" _compiled_local_file_id_pattern = re.compile(_LOCAL_FILE_ID_PATTERN) _compiled_remote_file_id_pattern = re.compile(_REMOTE_FILE_ID_PATTERN) @@ -59,3 +60,9 @@ def extract_local_file_ids(str_: str) -> List[str]: def extract_remote_file_ids(str_: str) -> List[str]: return _compiled_remote_file_id_pattern.findall(str_) + + +def generate_fake_remote_file_ids() -> Generator[str, None, None]: + counter = 0 + while True: + yield _REMOTE_FILE_ID_PREFIX + f"{counter:015d}" diff --git a/erniebot-agent/tests/integration_tests/chat_models/test_chat_model.py b/erniebot-agent/tests/integration_tests/chat_models/test_chat_model.py index 8c8f5ca4e..0932f3c97 100644 --- a/erniebot-agent/tests/integration_tests/chat_models/test_chat_model.py +++ b/erniebot-agent/tests/integration_tests/chat_models/test_chat_model.py @@ -1,4 +1,3 @@ -import os import unittest import pytest diff --git a/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py b/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py index 9731127c7..c07bfe8fb 100644 --- a/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py +++ b/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py @@ -1,80 +1,14 @@ import contextlib -import re -import uuid -import erniebot_agent.file_io.protocol as protocol +from erniebot_agent.file_io import protocol from erniebot_agent.file_io.remote_file import RemoteFile, RemoteFileClient -class FakeRemoteFileProtocol(object): - _UUID_PATTERN = r"[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" - _FILE_ID_PREFIX = r"file-fake-remote-" - _FILE_ID_PATTERN = _FILE_ID_PREFIX + _UUID_PATTERN - - _followed = False - - @classmethod - def is_remote_file_id(cls, str_): - return re.fullmatch(cls._FILE_ID_PATTERN, str_) is not None - - @classmethod - def extract_remote_file_ids(cls, str_): - return re.findall(cls._FILE_ID_PATTERN, str_) - - @classmethod - def generate_remote_file_id(cls): - return cls._FILE_ID_PREFIX + str(uuid.uuid4()) - - get_timestamp = protocol.get_timestamp - - @classmethod - @contextlib.contextmanager - def follow(cls, old_protocol=None): - if not cls._followed: - names_methods_to_monkey_patch = ( - "is_remote_file_id", - "extract_remote_file_ids", - "get_timestamp", - ) - - if old_protocol is None: - old_protocol = protocol - - _old_methods = {} - - for method_name in names_methods_to_monkey_patch: - old_method = getattr(old_protocol, method_name) - new_method = getattr(cls, method_name) - _old_methods[method_name] = old_method - setattr(old_protocol, method_name, new_method) - - cls._followed = True - - yield - - for method_name in names_methods_to_monkey_patch: - old_method = _old_methods[method_name] - setattr(old_protocol, method_name, old_method) - - cls._followed = False - - else: - yield - - class FakeRemoteFileClient(RemoteFileClient): - _protocol = FakeRemoteFileProtocol - def __init__(self, server): super().__init__() - if server.protocol is not self.protocol: - raise ValueError("Server and client do not share the same protocol.") self._server = server - @property - def protocol(self): - return self._protocol - @property def server(self): if not self._server.started: @@ -107,28 +41,22 @@ async def create_temporary_url(self, file_id, expire_after): raise RuntimeError("Method not supported") def _create_file_obj_from_dict(self, dict_): - with self._protocol.follow(): - return RemoteFile( - id=dict_["id"], - filename=dict_["filename"], - byte_size=dict_["byte_size"], - created_at=dict_["created_at"], - purpose=dict_["purpose"], - metadata=dict_["metadata"], - client=self, - ) + return RemoteFile( + id=dict_["id"], + filename=dict_["filename"], + byte_size=dict_["byte_size"], + created_at=dict_["created_at"], + purpose=dict_["purpose"], + metadata=dict_["metadata"], + client=self, + ) class FakeRemoteFileServer(object): - _protocol = FakeRemoteFileProtocol - def __init__(self): super().__init__() self._storage = None - - @property - def protocol(self): - return self._protocol + self._file_id_iter = protocol.generate_fake_remote_file_ids() @property def storage(self): @@ -139,10 +67,10 @@ def started(self): return self._storage is not None async def upload_file(self, file_path, file_purpose, file_metadata): - id_ = self._protocol.generate_remote_file_id() + id_ = next(self._file_id_iter) filename = file_path.name byte_size = file_path.stat().st_size - created_at = self._protocol.get_timestamp() + created_at = protocol.get_timestamp() with file_path.open("rb") as f: contents = f.read() file = dict( From f4f61e6239741fb6e2f82ce228ea04bc5a15a569 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Thu, 21 Dec 2023 21:19:31 +0800 Subject: [PATCH 11/63] Fix type hints --- erniebot-agent/src/erniebot_agent/file_io/base.py | 2 +- erniebot-agent/src/erniebot_agent/file_io/file_manager.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/file_io/base.py b/erniebot-agent/src/erniebot_agent/file_io/base.py index 9508e6028..3236ed04b 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/base.py +++ b/erniebot-agent/src/erniebot_agent/file_io/base.py @@ -53,7 +53,7 @@ def __repr__(self) -> str: async def read_contents(self) -> bytes: raise NotImplementedError - async def write_contents_to(self, local_path: Union[str, os.PathLike[str]]) -> None: + async def write_contents_to(self, local_path: Union[str, os.PathLike]) -> None: contents = await self.read_contents() await anyio.Path(local_path).write_bytes(contents) diff --git a/erniebot-agent/src/erniebot_agent/file_io/file_manager.py b/erniebot-agent/src/erniebot_agent/file_io/file_manager.py index 253a64ffa..865138cdf 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file_io/file_manager.py @@ -33,7 +33,7 @@ logger = logging.getLogger(__name__) -FilePath: TypeAlias = Union[str, os.PathLike[str]] +FilePath: TypeAlias = Union[str, os.PathLike] @final From 2860e8a3f871ea601afc3aa0ffe65a07eba67ccd Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Fri, 22 Dec 2023 14:00:01 +0800 Subject: [PATCH 12/63] merge from upstream --- erniebot-agent/src/erniebot_agent/{file_io => file}/__init__.py | 0 erniebot-agent/src/erniebot_agent/{file_io => file}/base.py | 0 erniebot-agent/src/erniebot_agent/{file_io => file}/factory.py | 0 .../src/erniebot_agent/{file_io => file}/file_manager.py | 0 .../src/erniebot_agent/{file_io => file}/file_registry.py | 0 erniebot-agent/src/erniebot_agent/{file_io => file}/local_file.py | 0 erniebot-agent/src/erniebot_agent/{file_io => file}/protocol.py | 0 .../src/erniebot_agent/{file_io => file}/remote_file.py | 0 8 files changed, 0 insertions(+), 0 deletions(-) rename erniebot-agent/src/erniebot_agent/{file_io => file}/__init__.py (100%) rename erniebot-agent/src/erniebot_agent/{file_io => file}/base.py (100%) rename erniebot-agent/src/erniebot_agent/{file_io => file}/factory.py (100%) rename erniebot-agent/src/erniebot_agent/{file_io => file}/file_manager.py (100%) rename erniebot-agent/src/erniebot_agent/{file_io => file}/file_registry.py (100%) rename erniebot-agent/src/erniebot_agent/{file_io => file}/local_file.py (100%) rename erniebot-agent/src/erniebot_agent/{file_io => file}/protocol.py (100%) rename erniebot-agent/src/erniebot_agent/{file_io => file}/remote_file.py (100%) diff --git a/erniebot-agent/src/erniebot_agent/file_io/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py similarity index 100% rename from erniebot-agent/src/erniebot_agent/file_io/__init__.py rename to erniebot-agent/src/erniebot_agent/file/__init__.py diff --git a/erniebot-agent/src/erniebot_agent/file_io/base.py b/erniebot-agent/src/erniebot_agent/file/base.py similarity index 100% rename from erniebot-agent/src/erniebot_agent/file_io/base.py rename to erniebot-agent/src/erniebot_agent/file/base.py diff --git a/erniebot-agent/src/erniebot_agent/file_io/factory.py b/erniebot-agent/src/erniebot_agent/file/factory.py similarity index 100% rename from erniebot-agent/src/erniebot_agent/file_io/factory.py rename to erniebot-agent/src/erniebot_agent/file/factory.py diff --git a/erniebot-agent/src/erniebot_agent/file_io/file_manager.py b/erniebot-agent/src/erniebot_agent/file/file_manager.py similarity index 100% rename from erniebot-agent/src/erniebot_agent/file_io/file_manager.py rename to erniebot-agent/src/erniebot_agent/file/file_manager.py diff --git a/erniebot-agent/src/erniebot_agent/file_io/file_registry.py b/erniebot-agent/src/erniebot_agent/file/file_registry.py similarity index 100% rename from erniebot-agent/src/erniebot_agent/file_io/file_registry.py rename to erniebot-agent/src/erniebot_agent/file/file_registry.py diff --git a/erniebot-agent/src/erniebot_agent/file_io/local_file.py b/erniebot-agent/src/erniebot_agent/file/local_file.py similarity index 100% rename from erniebot-agent/src/erniebot_agent/file_io/local_file.py rename to erniebot-agent/src/erniebot_agent/file/local_file.py diff --git a/erniebot-agent/src/erniebot_agent/file_io/protocol.py b/erniebot-agent/src/erniebot_agent/file/protocol.py similarity index 100% rename from erniebot-agent/src/erniebot_agent/file_io/protocol.py rename to erniebot-agent/src/erniebot_agent/file/protocol.py diff --git a/erniebot-agent/src/erniebot_agent/file_io/remote_file.py b/erniebot-agent/src/erniebot_agent/file/remote_file.py similarity index 100% rename from erniebot-agent/src/erniebot_agent/file_io/remote_file.py rename to erniebot-agent/src/erniebot_agent/file/remote_file.py From 0b02a1aeeb4f6b3c7c7916908883be960f36d9c6 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Fri, 22 Dec 2023 16:23:07 +0800 Subject: [PATCH 13/63] Fix --- erniebot-agent/requirements.txt | 1 + .../src/erniebot_agent/agents/base.py | 4 +- .../agents/callback/callback_manager.py | 2 +- .../langchain/chat_models/erniebot.py | 4 +- .../extensions/langchain/llms/erniebot.py | 2 +- .../erniebot_agent/file_io/file_manager.py | 15 ++++--- .../erniebot_agent/file_io/file_registry.py | 16 +------- .../file_io/global_file_manager.py | 28 +++---------- .../src/erniebot_agent/utils/temp_file.py | 39 ------------------- .../mocks/mock_remote_file_client_server.py | 21 ++++++++-- erniebot/src/erniebot/cli.py | 2 +- erniebot/src/erniebot/http_client.py | 2 +- .../src/erniebot/resources/chat_completion.py | 2 +- 13 files changed, 45 insertions(+), 93 deletions(-) delete mode 100644 erniebot-agent/src/erniebot_agent/utils/temp_file.py diff --git a/erniebot-agent/requirements.txt b/erniebot-agent/requirements.txt index 685004abf..8fb1fa71d 100644 --- a/erniebot-agent/requirements.txt +++ b/erniebot-agent/requirements.txt @@ -1,5 +1,6 @@ aiohttp anyio +asyncio-atexit # erniebot jinja2 langchain diff --git a/erniebot-agent/src/erniebot_agent/agents/base.py b/erniebot-agent/src/erniebot_agent/agents/base.py index 4ec89aebb..abb8435a7 100644 --- a/erniebot-agent/src/erniebot_agent/agents/base.py +++ b/erniebot-agent/src/erniebot_agent/agents/base.py @@ -163,10 +163,10 @@ def _parse_tool_args(self, tool_args: str) -> Dict[str, Any]: try: args_dict = json.loads(tool_args) except json.JSONDecodeError: - raise ValueError(f"`tool_args` cannot be parsed as JSON. `tool_args` is {tool_args}") + raise ValueError(f"`tool_args` cannot be parsed as JSON. `tool_args`: {tool_args}") if not isinstance(args_dict, dict): - raise ValueError(f"`tool_args` cannot be interpreted as a dict. It loads as {args_dict} ") + raise ValueError(f"`tool_args` cannot be interpreted as a dict. `tool_args`: {tool_args}") return args_dict async def _sniff_and_extract_files_from_args( diff --git a/erniebot-agent/src/erniebot_agent/agents/callback/callback_manager.py b/erniebot-agent/src/erniebot_agent/agents/callback/callback_manager.py index 17d9949bc..c167cea23 100644 --- a/erniebot-agent/src/erniebot_agent/agents/callback/callback_manager.py +++ b/erniebot-agent/src/erniebot_agent/agents/callback/callback_manager.py @@ -62,7 +62,7 @@ async def handle_event(self, event_type: EventType, *args: Any, **kwargs: Any) - for handler in self._handlers: callback = getattr(handler, callback_name, None) if not inspect.iscoroutinefunction(callback): - raise TypeError("Callback must be a coroutine function.") + raise RuntimeError("Callback must be a coroutine function.") await callback(*args, **kwargs) async def on_run_start(self, agent: Agent, prompt: str) -> None: diff --git a/erniebot-agent/src/erniebot_agent/extensions/langchain/chat_models/erniebot.py b/erniebot-agent/src/erniebot_agent/extensions/langchain/chat_models/erniebot.py index 92b026b6c..f1104c5ad 100644 --- a/erniebot-agent/src/erniebot_agent/extensions/langchain/chat_models/erniebot.py +++ b/erniebot-agent/src/erniebot_agent/extensions/langchain/chat_models/erniebot.py @@ -173,7 +173,7 @@ def _stream( **kwargs: Any, ) -> Iterator[ChatGenerationChunk]: if stop is not None: - raise TypeError("Currently, `stop` is not supported when streaming is enabled.") + raise ValueError("Currently, `stop` is not supported when streaming is enabled.") params = self._invocation_params params.update(kwargs) params["messages"] = self._convert_messages_to_dicts(messages) @@ -195,7 +195,7 @@ async def _astream( **kwargs: Any, ) -> AsyncIterator[ChatGenerationChunk]: if stop is not None: - raise TypeError("Currently, `stop` is not supported when streaming is enabled.") + raise ValueError("Currently, `stop` is not supported when streaming is enabled.") params = self._invocation_params params.update(kwargs) params["messages"] = self._convert_messages_to_dicts(messages) diff --git a/erniebot-agent/src/erniebot_agent/extensions/langchain/llms/erniebot.py b/erniebot-agent/src/erniebot_agent/extensions/langchain/llms/erniebot.py index ed4a75c24..a17827f62 100644 --- a/erniebot-agent/src/erniebot_agent/extensions/langchain/llms/erniebot.py +++ b/erniebot-agent/src/erniebot_agent/extensions/langchain/llms/erniebot.py @@ -149,7 +149,7 @@ def _stream( **kwargs: Any, ) -> Iterator[GenerationChunk]: if stop is not None: - raise TypeError("Currently, `stop` is not supported when streaming is enabled.") + raise ValueError("Currently, `stop` is not supported when streaming is enabled.") params = self._invocation_params params.update(kwargs) params["messages"] = [self._build_user_message_from_prompt(prompt)] diff --git a/erniebot-agent/src/erniebot_agent/file_io/file_manager.py b/erniebot-agent/src/erniebot_agent/file_io/file_manager.py index 865138cdf..a28afcd0f 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file_io/file_manager.py @@ -45,6 +45,7 @@ def __init__( remote_file_client: Optional[RemoteFileClient] = None, save_dir: Optional[FilePath] = None, *, + default_file_type: Optional[Literal["local", "remote"]] = None, prune_on_close: bool = True, ) -> None: super().__init__() @@ -56,6 +57,7 @@ def __init__( # This can be done lazily, but we need to be careful about race conditions. self._temp_dir = self._create_temp_dir() self._save_dir = pathlib.Path(self._temp_dir.name) + self._default_file_type = default_file_type self._prune_on_close = prune_on_close self._file_registry = FileRegistry() @@ -135,7 +137,7 @@ async def create_file_from_path( elif file_type == "remote": file = await self.create_remote_file_from_path(file_path, file_purpose, file_metadata) else: - raise ValueError(f"Unsupported file type: {file_type}") + raise RuntimeError(f"Unsupported file type: {file_type}") return file async def create_local_file_from_path( @@ -236,7 +238,7 @@ async def create_file_from_bytes( file_metadata, ) else: - raise ValueError(f"Unsupported file type: {file_type}") + raise RuntimeError(f"Unsupported file type: {file_type}") finally: if should_remove_file: await async_file_path.unlink() @@ -315,10 +317,13 @@ async def _create_remote_file_from_path( return file def _get_default_file_type(self) -> Literal["local", "remote"]: - if self._remote_file_client is not None: - return "remote" + if self._default_file_type is not None: + return self._default_file_type else: - return "local" + if self._remote_file_client is not None: + return "remote" + else: + return "local" def _get_unique_file_path( self, prefix: Optional[str] = None, suffix: Optional[str] = None diff --git a/erniebot-agent/src/erniebot_agent/file_io/file_registry.py b/erniebot-agent/src/erniebot_agent/file_io/file_registry.py index 79986b572..c8acb5001 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/file_registry.py +++ b/erniebot-agent/src/erniebot_agent/file_io/file_registry.py @@ -17,22 +17,8 @@ from erniebot_agent.file_io.base import File -class BaseFileRegistry(object): - def register_file(self, file: File, *, allow_overwrite: bool = False, check_type: bool = True) -> None: - raise NotImplementedError - - def unregister_file(self, file: File) -> None: - raise NotImplementedError - - def look_up_file(self, file_id: str) -> Optional[File]: - raise NotImplementedError - - def list_files(self) -> List[File]: - raise NotImplementedError - - @final -class FileRegistry(BaseFileRegistry): +class FileRegistry(object): def __init__(self) -> None: super().__init__() self._id_to_file: Dict[str, File] = {} diff --git a/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py b/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py index 5296bc95a..724b47ef4 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py @@ -12,17 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio -import atexit -from typing import Any, List, Optional +import asyncio_atexit +from typing import Any, Optional from erniebot_agent.file_io.file_manager import FileManager from erniebot_agent.file_io.remote_file import AIStudioFileClient from erniebot_agent.utils import config_from_environ as C -from erniebot_agent.utils.mixins import Closeable _global_file_manager: Optional[FileManager] = None -_objects_to_close: List[Closeable] = [] def get_global_file_manager() -> FileManager: @@ -46,6 +43,9 @@ def configure_global_file_manager( def _create_default_file_manager( access_token: Optional[str], save_dir: Optional[str], **opts: Any ) -> FileManager: + async def _close_file_manager(): + await file_manager.close() + if access_token is None: access_token = C.get_global_access_token() if save_dir is None: @@ -55,21 +55,5 @@ def _create_default_file_manager( else: remote_file_client = None file_manager = FileManager(remote_file_client, save_dir, **opts) - _objects_to_close.append(file_manager) + asyncio_atexit.register(_close_file_manager) return file_manager - - -def _close_objects(): - async def _close_objects_sequentially(): - for obj in _objects_to_close: - await obj.close() - - if _objects_to_close: - # Since async atexit is not officially supported by Python, - # we start a new event loop to do the cleanup. - asyncio.run(_close_objects_sequentially()) - _objects_to_close.clear() - - -# FIXME: The exit handler may not be called when using multiprocessing. -atexit.register(_close_objects) diff --git a/erniebot-agent/src/erniebot_agent/utils/temp_file.py b/erniebot-agent/src/erniebot_agent/utils/temp_file.py deleted file mode 100644 index ecb98ac43..000000000 --- a/erniebot-agent/src/erniebot_agent/utils/temp_file.py +++ /dev/null @@ -1,39 +0,0 @@ -# Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import atexit -import logging -import pathlib -import tempfile -from tempfile import TemporaryDirectory -from typing import Any, List - -logger = logging.getLogger(__name__) - -_tracked_temp_dirs: List[TemporaryDirectory] = [] - - -def create_tracked_temp_dir(*args: Any, **kwargs: Any) -> pathlib.Path: - # Borrowed from - # https://github.com/pypa/pipenv/blob/247a14369d300a6980a8dd634d9060bf6f582d2d/pipenv/utils/fileutils.py#L197 - def _cleanup() -> None: - try: - temp_dir.cleanup() - except Exception as e: - logger.warning("Failed to clean up temporary directory: %s", temp_dir.name, exc_info=e) - - temp_dir = tempfile.TemporaryDirectory(*args, **kwargs) - _tracked_temp_dirs.append(temp_dir) - atexit.register(_cleanup) - return pathlib.Path(temp_dir.name) diff --git a/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py b/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py index c07bfe8fb..e14819231 100644 --- a/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py +++ b/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py @@ -56,7 +56,7 @@ class FakeRemoteFileServer(object): def __init__(self): super().__init__() self._storage = None - self._file_id_iter = protocol.generate_fake_remote_file_ids() + self._file_id_iter = None @property def storage(self): @@ -67,6 +67,8 @@ def started(self): return self._storage is not None async def upload_file(self, file_path, file_purpose, file_metadata): + if not self.started: + raise ServerError("Server is not running.") id_ = next(self._file_id_iter) filename = file_path.name byte_size = file_path.stat().st_size @@ -86,12 +88,16 @@ async def upload_file(self, file_path, file_purpose, file_metadata): return file async def retrieve_file(self, file_id): + if not self.started: + raise ServerError("Server is not running.") try: return self._storage[file_id] except KeyError as e: raise ServerError("File not found") from e async def retrieve_file_contents(self, file_id): + if not self.started: + raise ServerError("Server is not running.") try: file = self._storage[file_id] except KeyError as e: @@ -100,9 +106,13 @@ async def retrieve_file_contents(self, file_id): return file["contents"] async def list_files(self): + if not self.started: + raise ServerError("Server is not running.") return list(self._storage.values()) async def delete_file(self, file_id) -> None: + if not self.started: + raise ServerError("Server is not running.") try: return self._storage[file_id] except KeyError as e: @@ -111,8 +121,13 @@ async def delete_file(self, file_id) -> None: @contextlib.contextmanager def start(self): self._storage = {} - yield self - self._storage = None + self._file_id_iter = protocol.generate_fake_remote_file_ids() + try: + yield self + finally: + self._storage = None + self._file_id_iter.close() + self._file_id_iter = None class ServerError(Exception): diff --git a/erniebot/src/erniebot/cli.py b/erniebot/src/erniebot/cli.py index c50c882c3..84c752b3f 100644 --- a/erniebot/src/erniebot/cli.py +++ b/erniebot/src/erniebot/cli.py @@ -118,7 +118,7 @@ def get_resource_class(cls): def register_api_to_parser(cls, parser, api_name): def _find_method(method_name): if not hasattr(cls, method_name): - raise AttributeError(f"{cls.__name__}.{method_name} is not found.") + raise RuntimeError(f"{cls.__name__}.{method_name} is not found.") method = getattr(cls, method_name) if not callable(method): raise RuntimeError(f"{cls.__name__}.{method_name} is not callable.") diff --git a/erniebot/src/erniebot/http_client.py b/erniebot/src/erniebot/http_client.py index e3043beca..9db0e510f 100644 --- a/erniebot/src/erniebot/http_client.py +++ b/erniebot/src/erniebot/http_client.py @@ -263,7 +263,7 @@ async def asend_request_raw( request_timeout: Optional[float], ) -> aiohttp.ClientResponse: if files is not None: - raise TypeError("`files` is currently not supported.") + raise ValueError("`files` is currently not supported.") timeout = aiohttp.ClientTimeout( total=request_timeout if request_timeout else self.DEFAULT_REQUEST_TIMEOUT_SECS diff --git a/erniebot/src/erniebot/resources/chat_completion.py b/erniebot/src/erniebot/resources/chat_completion.py index 9f88d541a..21c305383 100644 --- a/erniebot/src/erniebot/resources/chat_completion.py +++ b/erniebot/src/erniebot/resources/chat_completion.py @@ -481,7 +481,7 @@ def _set_val_if_key_exists(src: dict, dst: dict, key: str) -> None: if model == "ernie-turbo": for arg in ("functions", "stop", "disable_search", "enable_citation"): if arg in kwargs: - raise ValueError(f"`{arg}` is not supported by the {model} model.") + raise errors.InvalidArgumentError(f"`{arg}` is not supported by the {model} model.") params["messages"] = messages if "functions" in kwargs: functions = kwargs["functions"] From 82b63a5d0a15bf28446bbf26e7e33d6a6af04176 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Fri, 22 Dec 2023 16:24:29 +0800 Subject: [PATCH 14/63] Fix typing --- .../src/erniebot_agent/file_io/global_file_manager.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py b/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py index 724b47ef4..7a6afc329 100644 --- a/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file_io/global_file_manager.py @@ -12,9 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import asyncio_atexit from typing import Any, Optional +import asyncio_atexit # type: ignore + from erniebot_agent.file_io.file_manager import FileManager from erniebot_agent.file_io.remote_file import AIStudioFileClient from erniebot_agent.utils import config_from_environ as C From d54a93bead5e4cf1c7531297318ea021a3c2f146 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Fri, 22 Dec 2023 16:36:46 +0800 Subject: [PATCH 15/63] Fix style --- erniebot-agent/src/erniebot_agent/agents/base.py | 6 +++--- .../src/erniebot_agent/file/global_file_manager.py | 4 ++-- erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py | 7 ++++++- .../src/erniebot_agent/utils/config_from_environ.py | 5 ----- erniebot-agent/tests/unit_tests/file/test_local_file.py | 4 ++-- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/agents/base.py b/erniebot-agent/src/erniebot_agent/agents/base.py index 8886fdcb8..10646cd7e 100644 --- a/erniebot-agent/src/erniebot_agent/agents/base.py +++ b/erniebot-agent/src/erniebot_agent/agents/base.py @@ -27,11 +27,11 @@ ToolResponse, ) from erniebot_agent.chat_models.base import ChatModel -from erniebot_agent.file import protocol +from erniebot_agent.file import get_global_file_manager, protocol from erniebot_agent.file.base import File from erniebot_agent.file.file_manager import FileManager -from erniebot_agent.memory.base import Memory from erniebot_agent.memory import Message, SystemMessage +from erniebot_agent.memory.base import Memory from erniebot_agent.tools.base import BaseTool from erniebot_agent.tools.tool_manager import ToolManager from erniebot_agent.utils.mixins import GradioMixin @@ -77,7 +77,7 @@ def __init__( else: self._callback_manager = CallbackManager(callbacks) if file_manager is None: - file_manager = file.get_global_file_manager() + file_manager = get_global_file_manager() self.plugins = plugins self._file_manager = file_manager self._init_file_repr() diff --git a/erniebot-agent/src/erniebot_agent/file/global_file_manager.py b/erniebot-agent/src/erniebot_agent/file/global_file_manager.py index 7a6afc329..c4bfd329c 100644 --- a/erniebot-agent/src/erniebot_agent/file/global_file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/global_file_manager.py @@ -16,8 +16,8 @@ import asyncio_atexit # type: ignore -from erniebot_agent.file_io.file_manager import FileManager -from erniebot_agent.file_io.remote_file import AIStudioFileClient +from erniebot_agent.file.file_manager import FileManager +from erniebot_agent.file.remote_file import AIStudioFileClient from erniebot_agent.utils import config_from_environ as C _global_file_manager: Optional[FileManager] = None diff --git a/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py b/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py index 68ba69964..62cdbf8b3 100644 --- a/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py +++ b/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py @@ -14,7 +14,12 @@ from erniebot_agent.file import get_global_file_manager from erniebot_agent.file.file_manager import FileManager -from erniebot_agent.memory.messages import AIMessage, FunctionCall, HumanMessage, Message +from erniebot_agent.memory.messages import ( + AIMessage, + FunctionCall, + HumanMessage, + Message, +) from erniebot_agent.tools.remote_tool import RemoteTool, tool_registor from erniebot_agent.tools.schema import ( Endpoint, diff --git a/erniebot-agent/src/erniebot_agent/utils/config_from_environ.py b/erniebot-agent/src/erniebot_agent/utils/config_from_environ.py index 28cc48931..83b883fbb 100644 --- a/erniebot-agent/src/erniebot_agent/utils/config_from_environ.py +++ b/erniebot-agent/src/erniebot_agent/utils/config_from_environ.py @@ -15,14 +15,9 @@ import os from typing import Optional -<<<<<<<< HEAD:erniebot-agent/src/erniebot_agent/utils/config_from_environ.py def get_global_access_token() -> Optional[str]: return _get_val_from_env_var("EB_AGENT_ACCESS_TOKEN") -======== -from erniebot_agent.file.file_manager import FileManager -from erniebot_agent.file.remote_file import AIStudioFileClient ->>>>>>>> official:erniebot-agent/src/erniebot_agent/file/factory.py def get_global_save_dir() -> Optional[str]: diff --git a/erniebot-agent/tests/unit_tests/file/test_local_file.py b/erniebot-agent/tests/unit_tests/file/test_local_file.py index b06052ba3..52cbec45c 100644 --- a/erniebot-agent/tests/unit_tests/file/test_local_file.py +++ b/erniebot-agent/tests/unit_tests/file/test_local_file.py @@ -1,8 +1,8 @@ import pathlib import tempfile -import erniebot_agent.file_io.protocol as protocol -from erniebot_agent.file_io.local_file import LocalFile, create_local_file_from_path +import erniebot_agent.file.protocol as protocol +from erniebot_agent.file.local_file import LocalFile, create_local_file_from_path def test_create_local_file_from_path(): From 7df8462f49df1755b0855943ce45bc8c64da06f4 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Fri, 22 Dec 2023 16:46:55 +0800 Subject: [PATCH 16/63] Fix cleanup bugs --- .../src/erniebot_agent/file/global_file_manager.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/file/global_file_manager.py b/erniebot-agent/src/erniebot_agent/file/global_file_manager.py index c4bfd329c..eb20814e3 100644 --- a/erniebot-agent/src/erniebot_agent/file/global_file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/global_file_manager.py @@ -23,14 +23,14 @@ _global_file_manager: Optional[FileManager] = None -def get_global_file_manager() -> FileManager: +async def get_global_file_manager() -> FileManager: global _global_file_manager if _global_file_manager is None: - _global_file_manager = _create_default_file_manager(access_token=None, save_dir=None) + _global_file_manager = await _create_default_file_manager(access_token=None, save_dir=None) return _global_file_manager -def configure_global_file_manager( +async def configure_global_file_manager( access_token: Optional[str] = None, save_dir: Optional[str] = None, **opts: Any ) -> None: global _global_file_manager @@ -38,10 +38,10 @@ def configure_global_file_manager( raise RuntimeError( "The global file manager can only be configured once before calling `get_global_file_manager`." ) - _global_file_manager = _create_default_file_manager(access_token=access_token, save_dir=save_dir, **opts) + _global_file_manager = await _create_default_file_manager(access_token=access_token, save_dir=save_dir, **opts) -def _create_default_file_manager( +async def _create_default_file_manager( access_token: Optional[str], save_dir: Optional[str], **opts: Any ) -> FileManager: async def _close_file_manager(): From 286f2bd409ef3529fb1b758d6476e8b5989350e3 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Fri, 22 Dec 2023 16:53:20 +0800 Subject: [PATCH 17/63] temp add docstring --- .../src/erniebot_agent/file/__init__.py | 12 +- .../src/erniebot_agent/file/base.py | 31 ++ .../src/erniebot_agent/file/file_manager.py | 394 ++++++++++-------- .../file/global_file_manager.py | 12 +- 4 files changed, 269 insertions(+), 180 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/file/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py index d4273b94b..70b4e74ee 100644 --- a/erniebot-agent/src/erniebot_agent/file/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file/__init__.py @@ -12,7 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. -from erniebot_agent.file_io.global_file_manager import ( +""" +File module is used to manage the file system by a global file_manager. +Including `local file` and `remote file`. + +A few notes about the current state of this submodule: +- If you do not set environment variable `EB_ACCESS_TOKEN`, it will be under default setting. +- Method `configure_global_file_manager` can only be called once at the beginning. +- When you want to get file_manger, you can use method `get_global_file_manager`. +""" + +from erniebot_agent.file.global_file_manager import ( configure_global_file_manager, get_global_file_manager, ) diff --git a/erniebot-agent/src/erniebot_agent/file/base.py b/erniebot-agent/src/erniebot_agent/file/base.py index 3236ed04b..2e2c5d8e4 100644 --- a/erniebot-agent/src/erniebot_agent/file/base.py +++ b/erniebot-agent/src/erniebot_agent/file/base.py @@ -20,6 +20,23 @@ class File(metaclass=abc.ABCMeta): + """ + Abstract base class representing a generic file. + + Attributes: + id (str): Unique identifier for the file. + filename (str): File name. + byte_size (int): Size of the file in bytes. + created_at (str): Timestamp indicating the file creation time. + purpose (str): Purpose or use case of the file. [] + metadata (Dict[str, Any]): Additional metadata associated with the file. + + Methods: + read_contents: Abstract method to asynchronously read the file contents. + write_contents_to: Asynchronously write the file contents to a local path. + get_file_repr: Return a string representation for use in specific contexts. + to_dict: Convert the File object to a dictionary. + """ def __init__( self, *, @@ -30,6 +47,20 @@ def __init__( purpose: str, metadata: Dict[str, Any], ) -> None: + """ + Init method for the File class. + + Args: + id (str): Unique identifier for the file. + filename (str): File name. + byte_size (int): Size of the file in bytes. + created_at (str): Timestamp indicating the file creation time. + purpose (str): Purpose or use case of the file. [] + metadata (Dict[str, Any]): Additional metadata associated with the file. + + Returns: + None + """ super().__init__() self.id = id self.filename = filename diff --git a/erniebot-agent/src/erniebot_agent/file/file_manager.py b/erniebot-agent/src/erniebot_agent/file/file_manager.py index 865138cdf..9f521e04a 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/file_manager.py @@ -17,82 +17,121 @@ import pathlib import tempfile import uuid -from types import TracebackType -from typing import Any, Dict, List, Literal, Optional, Type, Union, final, overload +import weakref +from typing import Any, Dict, List, Literal, Optional, Union, overload import anyio -from typing_extensions import Self, TypeAlias +from typing_extensions import TypeAlias -from erniebot_agent.file_io import protocol -from erniebot_agent.file_io.base import File -from erniebot_agent.file_io.file_registry import FileRegistry -from erniebot_agent.file_io.local_file import LocalFile, create_local_file_from_path -from erniebot_agent.file_io.remote_file import RemoteFile, RemoteFileClient -from erniebot_agent.utils.exceptions import FileError -from erniebot_agent.utils.mixins import Closeable +from erniebot_agent.file.base import File +from erniebot_agent.file.file_registry import FileRegistry, get_file_registry +from erniebot_agent.file.local_file import LocalFile, create_local_file_from_path +from erniebot_agent.file.protocol import FilePurpose +from erniebot_agent.file.remote_file import RemoteFile, RemoteFileClient +from erniebot_agent.utils.exception import FileError logger = logging.getLogger(__name__) FilePath: TypeAlias = Union[str, os.PathLike] -@final -class FileManager(Closeable): - _temp_dir: Optional[tempfile.TemporaryDirectory] = None +class FileManager(object): + """ + Manages files, providing methods for creating, retrieving, and listing files. + + Attributes: + registry(FileRegistry): The file registry. + remote_file_client(RemoteFileClient): The remote file client. + save_dir (Optional[FilePath]): Directory for saving local files. + _file_registry (FileRegistry): Registry for keeping track of files. + + Methods: + __init__: Initialize the FileManager object. + + + create_file_from_path: Create a file from a specified file path. + create_local_file_from_path: Create a local file from a file path. + create_remote_file_from_path: Create a remote file from a file path. + create_file_from_bytes: Create a file from bytes. + retrieve_remote_file_by_id: Retrieve a remote file by its ID. + look_up_file_by_id: Look up a file by its ID. + list_remote_files: List remote files. + _fs_create_file: Create a file in the file system. + _fs_create_temp_dir: Create a temporary directory in the file system. + _clean_up_temp_dir: Clean up a temporary directory. + + """ + _remote_file_client: Optional[RemoteFileClient] def __init__( self, remote_file_client: Optional[RemoteFileClient] = None, - save_dir: Optional[FilePath] = None, *, - prune_on_close: bool = True, + auto_register: bool = True, + save_dir: Optional[FilePath] = None, ) -> None: - super().__init__() + """ + Initialize the FileManager object. - self._remote_file_client = remote_file_client + Args: + remote_file_client (Optional[RemoteFileClient]): The remote file client. + auto_register (bool): Automatically register files in the file registry. + save_dir (Optional[FilePath]): Directory for saving local files. + + Returns: + None + + """ + super().__init__() + if remote_file_client is not None: + self._remote_file_client = remote_file_client + else: + self._remote_file_client = None + self._auto_register = auto_register if save_dir is not None: self._save_dir = pathlib.Path(save_dir) else: # This can be done lazily, but we need to be careful about race conditions. - self._temp_dir = self._create_temp_dir() - self._save_dir = pathlib.Path(self._temp_dir.name) - self._prune_on_close = prune_on_close + self._save_dir = self._fs_create_temp_dir() + + self._file_registry = get_file_registry() + + @property + def registry(self) -> FileRegistry: + """ + Get the file registry. - self._file_registry = FileRegistry() - self._fully_managed_files: List[Union[LocalFile, RemoteFile]] = [] + Returns: + FileRegistry: The file registry. - self._closed = False + """ + return self._file_registry @property def remote_file_client(self) -> RemoteFileClient: + """ + Get the remote file client. + + Returns: + RemoteFileClient: The remote file client. + + Raises: + AttributeError: If no remote file client is set. + + """ if self._remote_file_client is None: raise AttributeError("No remote file client is set.") else: return self._remote_file_client - @property - def closed(self): - return self._closed - - async def __aenter__(self) -> Self: - return self - - async def __aexit__( - self, - exc_type: Optional[Type[BaseException]] = None, - exc_value: Optional[BaseException] = None, - traceback: Optional[TracebackType] = None, - ) -> None: - await self.close() - @overload async def create_file_from_path( self, file_path: FilePath, *, - file_purpose: protocol.FilePurpose = ..., + file_purpose: FilePurpose = ..., file_metadata: Optional[Dict[str, Any]] = ..., - file_type: Literal["local"], + file_type: Literal["local"] = ..., ) -> LocalFile: ... @@ -101,35 +140,42 @@ async def create_file_from_path( self, file_path: FilePath, *, - file_purpose: protocol.FilePurpose = ..., + file_purpose: FilePurpose = ..., file_metadata: Optional[Dict[str, Any]] = ..., file_type: Literal["remote"], ) -> RemoteFile: ... - @overload - async def create_file_from_path( - self, - file_path: FilePath, - *, - file_purpose: protocol.FilePurpose = ..., - file_metadata: Optional[Dict[str, Any]] = ..., - file_type: None = ..., - ) -> Union[LocalFile, RemoteFile]: - ... - async def create_file_from_path( self, file_path: FilePath, *, - file_purpose: protocol.FilePurpose = "assistants", + file_purpose: FilePurpose = "assistants", file_metadata: Optional[Dict[str, Any]] = None, file_type: Optional[Literal["local", "remote"]] = None, ) -> Union[LocalFile, RemoteFile]: - self.ensure_not_closed() + """ + Create a file from a specified file path. + + Args: + file_path (FilePath): The path to the file. + file_purpose (FilePurpose): The purpose or use case of the file. + file_metadata (Optional[Dict[str, Any]]): Additional metadata associated with the file. + file_type (Optional[Literal["local", "remote"]]): The type of file ("local" or "remote"). + + Returns: + Union[LocalFile, RemoteFile]: The created file. + + Raises: + ValueError: If an unsupported file type is provided. + + """ file: Union[LocalFile, RemoteFile] if file_type is None: - file_type = self._get_default_file_type() + if self._remote_file_client is not None: + file_type = "remote" + else: + file_type = "local" if file_type == "local": file = await self.create_local_file_from_path(file_path, file_purpose, file_metadata) elif file_type == "remote": @@ -141,10 +187,22 @@ async def create_file_from_path( async def create_local_file_from_path( self, file_path: FilePath, - file_purpose: protocol.FilePurpose, + file_purpose: FilePurpose, file_metadata: Optional[Dict[str, Any]], ) -> LocalFile: - file = await self._create_local_file_from_path( + """ + Create a local file from a local file path. + + Args: + file_path (FilePath): The path to the file. + file_purpose (FilePurpose): The purpose or use case of the file. + file_metadata (Optional[Dict[str, Any]]): Additional metadata associated with the file. + + Returns: + LocalFile: The created local file. + + """ + file = create_local_file_from_path( pathlib.Path(file_path), file_purpose, file_metadata or {}, @@ -153,18 +211,25 @@ async def create_local_file_from_path( return file async def create_remote_file_from_path( - self, - file_path: FilePath, - file_purpose: protocol.FilePurpose, - file_metadata: Optional[Dict[str, Any]], + self, file_path: FilePath, file_purpose: FilePurpose, file_metadata: Optional[Dict[str, Any]] ) -> RemoteFile: - file = await self._create_remote_file_from_path( - pathlib.Path(file_path), - file_purpose, - file_metadata, + """ + Create a remote file from a file path. + + Args: + file_path (FilePath): The path to the file. + file_purpose (FilePurpose): The purpose or use case of the file. + file_metadata (Optional[Dict[str, Any]]): Additional metadata associated with the file. + + Returns: + RemoteFile: The created remote file. + + """ + file = await self.remote_file_client.upload_file( + pathlib.Path(file_path), file_purpose, file_metadata or {} ) - self._file_registry.register_file(file) - self._fully_managed_files.append(file) + if self._auto_register: + self._file_registry.register_file(file) return file @overload @@ -173,9 +238,9 @@ async def create_file_from_bytes( file_contents: bytes, filename: str, *, - file_purpose: protocol.FilePurpose = ..., + file_purpose: FilePurpose = ..., file_metadata: Optional[Dict[str, Any]] = ..., - file_type: Literal["local"], + file_type: Literal["local"] = ..., ) -> LocalFile: ... @@ -185,155 +250,130 @@ async def create_file_from_bytes( file_contents: bytes, filename: str, *, - file_purpose: protocol.FilePurpose = ..., + file_purpose: FilePurpose = ..., file_metadata: Optional[Dict[str, Any]] = ..., file_type: Literal["remote"], ) -> RemoteFile: ... - @overload - async def create_file_from_bytes( - self, - file_contents: bytes, - filename: str, - *, - file_purpose: protocol.FilePurpose = ..., - file_metadata: Optional[Dict[str, Any]] = ..., - file_type: None = ..., - ) -> Union[LocalFile, RemoteFile]: - ... - async def create_file_from_bytes( self, file_contents: bytes, filename: str, *, - file_purpose: protocol.FilePurpose = "assistants", + file_purpose: FilePurpose = "assistants", file_metadata: Optional[Dict[str, Any]] = None, file_type: Optional[Literal["local", "remote"]] = None, ) -> Union[LocalFile, RemoteFile]: - self.ensure_not_closed() - if file_type is None: - file_type = self._get_default_file_type() - file_path = self._get_unique_file_path( - prefix=pathlib.PurePath(filename).stem, - suffix=pathlib.PurePath(filename).suffix, + """ + Create a file from bytes. + + Args: + file_contents (bytes): The content bytes of the file. + filename (str): The name of the file. + file_purpose (FilePurpose): The purpose or use case of the file. + file_metadata (Optional[Dict[str, Any]]): Additional metadata associated with the file. + file_type (Optional[Literal["local", "remote"]]): The type of file ("local" or "remote"). + + Returns: + Union[LocalFile, RemoteFile]: The created file. + + """ + # Can we do this with in-memory files? + file_path = await self._fs_create_file( + prefix=pathlib.PurePath(filename).stem, suffix=pathlib.PurePath(filename).suffix ) - async_file_path = anyio.Path(file_path) - await async_file_path.touch() - should_remove_file = True try: - async with await async_file_path.open("wb") as f: + async with await file_path.open("wb") as f: await f.write(file_contents) - file: Union[LocalFile, RemoteFile] - if file_type == "local": - file = await self._create_local_file_from_path(file_path, file_purpose, file_metadata) - should_remove_file = False - elif file_type == "remote": - file = await self._create_remote_file_from_path( - file_path, - file_purpose, - file_metadata, - ) - else: - raise ValueError(f"Unsupported file type: {file_type}") + if file_type is None: + if self._remote_file_client is not None: + file_type = "remote" + else: + file_type = "local" + file = await self.create_file_from_path( + file_path, + file_purpose=file_purpose, + file_metadata=file_metadata, + file_type=file_type, + ) finally: - if should_remove_file: - await async_file_path.unlink() - self._file_registry.register_file(file) - self._fully_managed_files.append(file) + if file_type == "remote": + await file_path.unlink() return file async def retrieve_remote_file_by_id(self, file_id: str) -> RemoteFile: - self.ensure_not_closed() + """ + Retrieve a remote file by its ID. + + Args: + file_id (str): The ID of the remote file. + + Returns: + RemoteFile: The retrieved remote file. + + """ file = await self.remote_file_client.retrieve_file(file_id) - self._file_registry.register_file(file) + if self._auto_register: + self._file_registry.register_file(file, allow_overwrite=True) return file - async def list_remote_files(self) -> List[RemoteFile]: - self.ensure_not_closed() - files = await self.remote_file_client.list_files() - return files - def look_up_file_by_id(self, file_id: str) -> Optional[File]: - self.ensure_not_closed() + """ + Look up a file by its ID. + + Args: + file_id (str): The ID of the file. + + Returns: + Optional[Union[LocalFile, RemoteFile]]: The looked-up file, or None if not found. + + Raises: + FileError: If the file with the specified ID is not found. + + """ file = self._file_registry.look_up_file(file_id) if file is None: raise FileError( - f"File with ID {repr(file_id)} not found. " - "Please check if `file_id` is correct and the file is registered." + f"File with ID '{file_id}' not found. " + "Please check if the file exists and the `file_id` is correct." ) return file - def list_registered_files(self) -> List[File]: - self.ensure_not_closed() - return self._file_registry.list_files() - - async def prune(self) -> None: - for file in self._fully_managed_files: - if isinstance(file, RemoteFile): - # FIXME: Currently this is not supported. - # await file.delete() - pass - elif isinstance(file, LocalFile): - assert self._save_dir in file.path.parents - await anyio.Path(file.path).unlink() - else: - raise AssertionError("Unexpected file type") - self._file_registry.unregister_file(file) - self._fully_managed_files.clear() - - async def close(self) -> None: - if not self._closed: - if self._remote_file_client is not None: - await self._remote_file_client.close() - if self._prune_on_close: - await self.prune() - if self._temp_dir is not None: - self._clean_up_temp_dir(self._temp_dir) - self._closed = True - - async def _create_local_file_from_path( - self, - file_path: pathlib.Path, - file_purpose: protocol.FilePurpose, - file_metadata: Optional[Dict[str, Any]], - ) -> LocalFile: - return create_local_file_from_path( - pathlib.Path(file_path), - file_purpose, - file_metadata or {}, - ) + async def list_remote_files(self) -> List[RemoteFile]: + """ + List remote files. - async def _create_remote_file_from_path( - self, - file_path: pathlib.Path, - file_purpose: protocol.FilePurpose, - file_metadata: Optional[Dict[str, Any]], - ) -> RemoteFile: - file = await self.remote_file_client.upload_file(file_path, file_purpose, file_metadata or {}) - return file + Returns: + List[RemoteFile]: The list of remote files. - def _get_default_file_type(self) -> Literal["local", "remote"]: - if self._remote_file_client is not None: - return "remote" - else: - return "local" + """ + files = await self.remote_file_client.list_files() + if self._auto_register: + for file in files: + self._file_registry.register_file(file, allow_overwrite=True) + return files - def _get_unique_file_path( + async def _fs_create_file( self, prefix: Optional[str] = None, suffix: Optional[str] = None - ) -> pathlib.Path: + ) -> anyio.Path: + """Create a file in the file system.""" filename = f"{prefix or ''}{str(uuid.uuid4())}{suffix or ''}" - file_path = self._save_dir / filename + file_path = anyio.Path(self._save_dir / filename) + await file_path.touch() return file_path - @staticmethod - def _create_temp_dir() -> tempfile.TemporaryDirectory: + def _fs_create_temp_dir(self) -> pathlib.Path: + """Create a temporary directory in the file system.""" temp_dir = tempfile.TemporaryDirectory() - return temp_dir + # The temporary directory shall be cleaned up when the file manager is + # garbage collected. + weakref.finalize(self, self._clean_up_temp_dir, temp_dir) + return pathlib.Path(temp_dir.name) @staticmethod def _clean_up_temp_dir(temp_dir: tempfile.TemporaryDirectory) -> None: + """Clean up a temporary directory.""" try: temp_dir.cleanup() except Exception as e: diff --git a/erniebot-agent/src/erniebot_agent/file/global_file_manager.py b/erniebot-agent/src/erniebot_agent/file/global_file_manager.py index 5296bc95a..8dff14770 100644 --- a/erniebot-agent/src/erniebot_agent/file/global_file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/global_file_manager.py @@ -1,3 +1,11 @@ +''' +Author: Southpika 513923576@qq.com +Date: 2023-12-22 14:20:41 +LastEditors: Southpika 513923576@qq.com +LastEditTime: 2023-12-22 15:22:39 +FilePath: /ERINE/ERNIE-Bot-SDK/erniebot-agent/src/erniebot_agent/file/global_file_manager.py +Description: 这是默认设置,请设置`customMade`, 打开koroFileHeader查看配置 进行设置: https://github.com/OBKoro1/koro1FileHeader/wiki/%E9%85%8D%E7%BD%AE +''' # Copyright (c) 2023 PaddlePaddle Authors. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,8 +24,8 @@ import atexit from typing import Any, List, Optional -from erniebot_agent.file_io.file_manager import FileManager -from erniebot_agent.file_io.remote_file import AIStudioFileClient +from erniebot_agent.file.file_manager import FileManager +from erniebot_agent.file.remote_file import AIStudioFileClient from erniebot_agent.utils import config_from_environ as C from erniebot_agent.utils.mixins import Closeable From dc5024a836de23824e8426c829e7d5560aa02afb Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Fri, 22 Dec 2023 17:12:53 +0800 Subject: [PATCH 18/63] Fix data race --- erniebot-agent/examples/cv_agent/CV_agent.py | 4 ++-- .../examples/plugins/multiple_plugins.py | 10 ++++---- erniebot-agent/examples/rpg_game_agent.py | 5 +--- .../src/erniebot_agent/agents/base.py | 23 ++++++++++--------- .../file/global_file_manager.py | 18 +++++++++------ .../src/erniebot_agent/tools/remote_tool.py | 20 ++++++++++++---- .../erniebot_agent/tools/remote_toolkit.py | 4 ---- .../apihub/test_pp_shituv2.py | 20 ++++++++-------- 8 files changed, 57 insertions(+), 47 deletions(-) diff --git a/erniebot-agent/examples/cv_agent/CV_agent.py b/erniebot-agent/examples/cv_agent/CV_agent.py index 6a397da70..231619442 100644 --- a/erniebot-agent/examples/cv_agent/CV_agent.py +++ b/erniebot-agent/examples/cv_agent/CV_agent.py @@ -2,7 +2,7 @@ from erniebot_agent.agents.functional_agent import FunctionalAgent from erniebot_agent.chat_models.erniebot import ERNIEBot -from erniebot_agent.file_io import ( +from erniebot_agent.file import ( configure_global_file_manager, get_global_file_manager, ) @@ -25,7 +25,7 @@ def __init__(self): async def run_agent(): - file_manager = get_global_file_manager() + file_manager = await get_global_file_manager() seg_file = await file_manager.create_file_from_path(file_path="cityscapes_demo.png", file_type="local") clas_file = await file_manager.create_file_from_path(file_path="class_img.jpg", file_type="local") ocr_file = await file_manager.create_file_from_path(file_path="ch.png", file_type="local") diff --git a/erniebot-agent/examples/plugins/multiple_plugins.py b/erniebot-agent/examples/plugins/multiple_plugins.py index 29d628813..a64dfbd43 100644 --- a/erniebot-agent/examples/plugins/multiple_plugins.py +++ b/erniebot-agent/examples/plugins/multiple_plugins.py @@ -6,9 +6,9 @@ from erniebot_agent.agents.callback.default import get_no_ellipsis_callback from erniebot_agent.agents.functional_agent import FunctionalAgent from erniebot_agent.chat_models.erniebot import ERNIEBot -from erniebot_agent.file_io import get_global_file_manager +from erniebot_agent.file import get_global_file_manager from erniebot_agent.memory.sliding_window_memory import SlidingWindowMemory -from erniebot_agent.messages import AIMessage, HumanMessage, Message +from erniebot_agent.memory import AIMessage, HumanMessage, Message from erniebot_agent.tools.base import Tool from erniebot_agent.tools.calculator_tool import CalculatorTool from erniebot_agent.tools.schema import ToolParameterView @@ -32,7 +32,7 @@ async def __call__(self, input_file_id: str, repeat_times: int) -> Dict[str, Any if "" in input_file_id: input_file_id = input_file_id.split("")[0] - file_manager = get_global_file_manager() + file_manager = await get_global_file_manager() input_file = file_manager.look_up_file_by_id(input_file_id) if input_file is None: raise RuntimeError("File not found") @@ -109,20 +109,20 @@ def examples(self) -> List[Message]: # TODO(shiyutang): replace this when model is online llm = ERNIEBot(model="ernie-3.5", api_type="custom") memory = SlidingWindowMemory(max_round=1) -file_manager = get_global_file_manager() # plugins = ["ChatFile", "eChart"] plugins: List[str] = [] agent = FunctionalAgent( llm=llm, tools=[TextRepeaterTool(), TextRepeaterNoFileTool(), CalculatorTool()], memory=memory, - file_manager=file_manager, callbacks=get_no_ellipsis_callback(), plugins=plugins, ) async def run_agent(): + file_manager = await get_global_file_manager() + docx_file = await file_manager.create_file_from_path( file_path="浅谈牛奶的营养与消费趋势.docx", file_type="remote", diff --git a/erniebot-agent/examples/rpg_game_agent.py b/erniebot-agent/examples/rpg_game_agent.py index 2c9aecfba..4a03edc70 100644 --- a/erniebot-agent/examples/rpg_game_agent.py +++ b/erniebot-agent/examples/rpg_game_agent.py @@ -24,9 +24,7 @@ from erniebot_agent.agents.base import Agent from erniebot_agent.agents.schema import AgentFile, AgentResponse from erniebot_agent.chat_models.erniebot import ERNIEBot -from erniebot_agent.file_io import get_global_file_manager -from erniebot_agent.file_io.base import File -from erniebot_agent.file_io.file_manager import FileManager +from erniebot_agent.file.base import File from erniebot_agent.memory.sliding_window_memory import SlidingWindowMemory from erniebot_agent.messages import AIMessage, HumanMessage, SystemMessage from erniebot_agent.tools.base import BaseTool @@ -87,7 +85,6 @@ def __init__( tools=tools, system_message=system_message, ) - self.file_manager: FileManager = get_global_file_manager() async def handle_tool(self, tool_name: str, tool_args: str) -> str: tool_response = await self._async_run_tool( diff --git a/erniebot-agent/src/erniebot_agent/agents/base.py b/erniebot-agent/src/erniebot_agent/agents/base.py index 10646cd7e..623480b5d 100644 --- a/erniebot-agent/src/erniebot_agent/agents/base.py +++ b/erniebot-agent/src/erniebot_agent/agents/base.py @@ -76,18 +76,16 @@ def __init__( self._callback_manager = callbacks else: self._callback_manager = CallbackManager(callbacks) - if file_manager is None: - file_manager = get_global_file_manager() - self.plugins = plugins self._file_manager = file_manager + self._plugins = plugins self._init_file_repr() def _init_file_repr(self): self.file_needs_url = False - if self.plugins: + if self._plugins: PLUGIN_WO_FILE = ["eChart"] - for plugin in self.plugins: + for plugin in self._plugins: if plugin not in PLUGIN_WO_FILE: self.file_needs_url = True @@ -175,12 +173,8 @@ async def _sniff_and_extract_files_from_args( for val in args.values(): if isinstance(val, str): if protocol.is_file_id(val): - if self._file_manager is None: - logger.warning( - f"A file is used by {repr(tool)}, but the agent has no file manager to fetch it." - ) - continue - file = self._file_manager.look_up_file_by_id(val) + file_manager = await self._get_file_manager() + file = file_manager.look_up_file_by_id(val) if file is None: raise RuntimeError(f"Unregistered file with ID {repr(val)} is used by {repr(tool)}.") agent_files.append(AgentFile(file=file, type=file_type, used_by=tool.tool_name)) @@ -190,3 +184,10 @@ async def _sniff_and_extract_files_from_args( for item in val: agent_files.extend(await self._sniff_and_extract_files_from_args(item, tool, file_type)) return agent_files + + async def _get_file_manager(self) -> FileManager: + if self._file_manager is None: + file_manager = await get_global_file_manager() + else: + file_manager = self._file_manager + return file_manager diff --git a/erniebot-agent/src/erniebot_agent/file/global_file_manager.py b/erniebot-agent/src/erniebot_agent/file/global_file_manager.py index eb20814e3..4b17c834f 100644 --- a/erniebot-agent/src/erniebot_agent/file/global_file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/global_file_manager.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import asyncio from typing import Any, Optional import asyncio_atexit # type: ignore @@ -21,12 +22,14 @@ from erniebot_agent.utils import config_from_environ as C _global_file_manager: Optional[FileManager] = None +_lock = asyncio.Lock() async def get_global_file_manager() -> FileManager: global _global_file_manager - if _global_file_manager is None: - _global_file_manager = await _create_default_file_manager(access_token=None, save_dir=None) + async with _lock: + if _global_file_manager is None: + _global_file_manager = await _create_default_file_manager(access_token=None, save_dir=None) return _global_file_manager @@ -34,11 +37,12 @@ async def configure_global_file_manager( access_token: Optional[str] = None, save_dir: Optional[str] = None, **opts: Any ) -> None: global _global_file_manager - if _global_file_manager is not None: - raise RuntimeError( - "The global file manager can only be configured once before calling `get_global_file_manager`." - ) - _global_file_manager = await _create_default_file_manager(access_token=access_token, save_dir=save_dir, **opts) + async with _lock: + if _global_file_manager is not None: + raise RuntimeError( + "The global file manager can only be configured once before calling `get_global_file_manager`." + ) + _global_file_manager = await _create_default_file_manager(access_token=access_token, save_dir=save_dir, **opts) async def _create_default_file_manager( diff --git a/erniebot-agent/src/erniebot_agent/tools/remote_tool.py b/erniebot-agent/src/erniebot_agent/tools/remote_tool.py index ef3ab1a8d..06b387ce6 100644 --- a/erniebot-agent/src/erniebot_agent/tools/remote_tool.py +++ b/erniebot-agent/src/erniebot_agent/tools/remote_tool.py @@ -4,6 +4,7 @@ import json from copy import deepcopy from typing import Any, Dict, List, Optional, Type +from erniebot_agent.file import get_global_file_manager import requests @@ -45,7 +46,7 @@ def __init__( server_url: str, headers: dict, version: str, - file_manager: FileManager, + file_manager: Optional[FileManager], examples: Optional[List[Message]] = None, tool_name_prefix: Optional[str] = None, ) -> None: @@ -86,11 +87,13 @@ async def fileid_to_byte(file_id, file_manager): async def convert_to_file_data(file_data: str, format: str): value = file_data.replace("", "").replace("", "") - byte_value = await fileid_to_byte(value, self.file_manager) + byte_value = await fileid_to_byte(value, file_manager) if format == "byte": byte_value = base64.b64encode(byte_value).decode() return byte_value + file_manager = await self._get_file_manager() + # 1. replace fileid with byte string parameter_file_info = get_file_info_from_param_view(self.tool_view.parameters) for key in tool_arguments.keys(): @@ -203,19 +206,21 @@ async def send_request(self, tool_arguments: Dict[str, Any]) -> dict: if len(returns_file_infos) == 0 and is_json_response(response): return response.json() + file_manager = await self._get_file_manager() + file_metadata = {"tool_name": self.tool_name} if is_json_response(response) and len(returns_file_infos) > 0: response_json = response.json() file_info = await parse_file_from_json_response( response_json, - file_manager=self.file_manager, + file_manager=file_manager, param_view=self.tool_view.returns, # type: ignore tool_name=self.tool_name, ) response_json.update(file_info) return response_json file = await parse_file_from_response( - response, self.file_manager, file_infos=returns_file_infos, file_metadata=file_metadata + response, file_manager, file_infos=returns_file_infos, file_metadata=file_metadata ) if file is not None: @@ -293,6 +298,13 @@ def __adhoc_post_process__(self, tool_response: dict) -> dict: result.pop(key) return tool_response + async def _get_file_manager(self) -> FileManager: + if self.file_manager is None: + file_manager = await get_global_file_manager() + else: + file_manager = self.file_manager + return file_manager + class RemoteToolRegistor: def __init__(self) -> None: diff --git a/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py b/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py index 62cdbf8b3..94f7e60c1 100644 --- a/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py +++ b/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py @@ -12,7 +12,6 @@ from openapi_spec_validator.readers import read_from_filename from yaml import safe_dump -from erniebot_agent.file import get_global_file_manager from erniebot_agent.file.file_manager import FileManager from erniebot_agent.memory.messages import ( AIMessage, @@ -201,9 +200,6 @@ def from_openapi_dict( ) ) - if file_manager is None: - file_manager = get_global_file_manager() - return RemoteToolkit( openapi=openapi_dict["openapi"], info=info, diff --git a/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py b/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py index bdf8262dd..d57373e86 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py @@ -1,8 +1,8 @@ from __future__ import annotations +from erniebot_agent.file.file_manager import FileManager import pytest -from erniebot_agent.file_io import get_global_file_manager from erniebot_agent.tools import RemoteToolkit from .base import RemoteToolTesting @@ -17,13 +17,13 @@ async def test_pp_shituv2(self): agent = self.get_agent(toolkit) - file_manager = get_global_file_manager() - file_path = self.download_file( - "https://paddlenlp.bj.bcebos.com/ebagent/ci/fixtures/remote-tools/pp_shituv2_input_img.png" - ) - file = await file_manager.create_file_from_path(file_path) + async with FileManager() as file_manager: + file_path = self.download_file( + "https://paddlenlp.bj.bcebos.com/ebagent/ci/fixtures/remote-tools/pp_shituv2_input_img.png" + ) + file = await file_manager.create_file_from_path(file_path) - result = await agent.async_run("对这张图片进行通用识别，包含的文件为：", files=[file]) - self.assertEqual(len(result.files), 2) - self.assertEqual(len(result.actions), 1) - self.assertIn("file-", result.text) + result = await agent.async_run("对这张图片进行通用识别，包含的文件为：", files=[file]) + self.assertEqual(len(result.files), 2) + self.assertEqual(len(result.actions), 1) + self.assertIn("file-", result.text) From fbec0a01dbe086590db79882a17c9245deab3332 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Fri, 22 Dec 2023 17:40:02 +0800 Subject: [PATCH 19/63] add docstring(part) --- .../src/erniebot_agent/file/base.py | 9 ++--- .../src/erniebot_agent/file/file_manager.py | 36 ++++++++----------- 2 files changed, 19 insertions(+), 26 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/file/base.py b/erniebot-agent/src/erniebot_agent/file/base.py index 2e2c5d8e4..d9b509258 100644 --- a/erniebot-agent/src/erniebot_agent/file/base.py +++ b/erniebot-agent/src/erniebot_agent/file/base.py @@ -22,7 +22,7 @@ class File(metaclass=abc.ABCMeta): """ Abstract base class representing a generic file. - + Attributes: id (str): Unique identifier for the file. filename (str): File name. @@ -36,7 +36,8 @@ class File(metaclass=abc.ABCMeta): write_contents_to: Asynchronously write the file contents to a local path. get_file_repr: Return a string representation for use in specific contexts. to_dict: Convert the File object to a dictionary. - """ + """ + def __init__( self, *, @@ -49,7 +50,7 @@ def __init__( ) -> None: """ Init method for the File class. - + Args: id (str): Unique identifier for the file. filename (str): File name. @@ -57,7 +58,7 @@ def __init__( created_at (str): Timestamp indicating the file creation time. purpose (str): Purpose or use case of the file. [] metadata (Dict[str, Any]): Additional metadata associated with the file. - + Returns: None """ diff --git a/erniebot-agent/src/erniebot_agent/file/file_manager.py b/erniebot-agent/src/erniebot_agent/file/file_manager.py index f113c16e3..4c78db396 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/file_manager.py @@ -36,17 +36,15 @@ FilePath: TypeAlias = Union[str, os.PathLike] -class FileManager(object): +class FileManager(Closeable): """ Manages files, providing methods for creating, retrieving, and listing files. Attributes: - registry(FileRegistry): The file registry. remote_file_client(RemoteFileClient): The remote file client. save_dir (Optional[FilePath]): Directory for saving local files. - _file_registry (FileRegistry): Registry for keeping track of files. - Methods: + Methods: create_file_from_path: Create a file from a specified file path. create_local_file_from_path: Create a local file from a file path. create_remote_file_from_path: Create a remote file from a file path. @@ -54,19 +52,18 @@ class FileManager(object): retrieve_remote_file_by_id: Retrieve a remote file by its ID. look_up_file_by_id: Look up a file by its ID. list_remote_files: List remote files. - _fs_create_file: Create a file in the file system. - _fs_create_temp_dir: Create a temporary directory in the file system. - _clean_up_temp_dir: Clean up a temporary directory. """ - _remote_file_client: Optional[RemoteFileClient] + + _temp_dir: Optional[tempfile.TemporaryDirectory] = None def __init__( self, remote_file_client: Optional[RemoteFileClient] = None, - *, - auto_register: bool = True, save_dir: Optional[FilePath] = None, + *, + default_file_type: Optional[Literal["local", "remote"]] = None, + prune_on_close: bool = True, ) -> None: """ Initialize the FileManager object. @@ -81,11 +78,8 @@ def __init__( """ super().__init__() - if remote_file_client is not None: - self._remote_file_client = remote_file_client - else: - self._remote_file_client = None - self._auto_register = auto_register + + self._remote_file_client = remote_file_client if save_dir is not None: self._save_dir = pathlib.Path(save_dir) else: @@ -159,7 +153,7 @@ async def create_file_from_path( self, file_path: FilePath, *, - file_purpose: FilePurpose = "assistants", + file_purpose: protocol.FilePurpose = "assistants", file_metadata: Optional[Dict[str, Any]] = None, file_type: Optional[Literal["local", "remote"]] = None, ) -> Union[LocalFile, RemoteFile]: @@ -179,12 +173,10 @@ async def create_file_from_path( ValueError: If an unsupported file type is provided. """ + self.ensure_not_closed() file: Union[LocalFile, RemoteFile] if file_type is None: - if self._remote_file_client is not None: - file_type = "remote" - else: - file_type = "local" + file_type = self._get_default_file_type() if file_type == "local": file = await self.create_local_file_from_path(file_path, file_purpose, file_metadata) elif file_type == "remote": @@ -196,7 +188,7 @@ async def create_file_from_path( async def create_local_file_from_path( self, file_path: FilePath, - file_purpose: FilePurpose, + file_purpose: protocol.FilePurpose, file_metadata: Optional[Dict[str, Any]], ) -> LocalFile: """ @@ -211,7 +203,7 @@ async def create_local_file_from_path( LocalFile: The created local file. """ - file = create_local_file_from_path( + file = await self._create_local_file_from_path( pathlib.Path(file_path), file_purpose, file_metadata or {}, From 866ba6db6215e9515c457fda080145f6ff45b1ae Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Fri, 22 Dec 2023 17:46:20 +0800 Subject: [PATCH 20/63] Fix bugs --- erniebot-agent/examples/cv_agent/CV_agent.py | 19 ++--- .../examples/plugins/multiple_plugins.py | 10 +-- .../src/erniebot_agent/agents/base.py | 4 +- .../erniebot_agent/agents/functional_agent.py | 2 +- .../src/erniebot_agent/file/__init__.py | 5 +- .../file/global_file_manager.py | 84 +++++++++++-------- .../src/erniebot_agent/tools/remote_tool.py | 4 +- .../erniebot_agent/tools/remote_toolkit.py | 2 +- .../src/erniebot_agent/utils/mixins.py | 19 +++-- erniebot-agent/test_concurrent.py | 10 +++ .../apihub/test_pp_shituv2.py | 2 +- 11 files changed, 92 insertions(+), 69 deletions(-) create mode 100644 erniebot-agent/test_concurrent.py diff --git a/erniebot-agent/examples/cv_agent/CV_agent.py b/erniebot-agent/examples/cv_agent/CV_agent.py index 231619442..b79281900 100644 --- a/erniebot-agent/examples/cv_agent/CV_agent.py +++ b/erniebot-agent/examples/cv_agent/CV_agent.py @@ -2,10 +2,7 @@ from erniebot_agent.agents.functional_agent import FunctionalAgent from erniebot_agent.chat_models.erniebot import ERNIEBot -from erniebot_agent.file import ( - configure_global_file_manager, - get_global_file_manager, -) +from erniebot_agent.file import GlobalFileManager from erniebot_agent.memory.whole_memory import WholeMemory from erniebot_agent.tools import RemoteToolkit @@ -17,15 +14,15 @@ def __init__(self): self.tools = self.toolkit.get_tools() -configure_global_file_manager(access_token="") -llm = ERNIEBot(model="ernie-bot", api_type="aistudio", access_token="") -toolkit = CVToolkit() -memory = WholeMemory() -agent = FunctionalAgent(llm=llm, tools=toolkit.tools, memory=memory) +async def run_agent(): + await GlobalFileManager().configure(access_token="") + llm = ERNIEBot(model="ernie-bot", api_type="aistudio", access_token="") + toolkit = CVToolkit() + memory = WholeMemory() + agent = FunctionalAgent(llm=llm, tools=toolkit.tools, memory=memory) -async def run_agent(): - file_manager = await get_global_file_manager() + file_manager = await GlobalFileManager().get() seg_file = await file_manager.create_file_from_path(file_path="cityscapes_demo.png", file_type="local") clas_file = await file_manager.create_file_from_path(file_path="class_img.jpg", file_type="local") ocr_file = await file_manager.create_file_from_path(file_path="ch.png", file_type="local") diff --git a/erniebot-agent/examples/plugins/multiple_plugins.py b/erniebot-agent/examples/plugins/multiple_plugins.py index a64dfbd43..411a9f265 100644 --- a/erniebot-agent/examples/plugins/multiple_plugins.py +++ b/erniebot-agent/examples/plugins/multiple_plugins.py @@ -6,9 +6,9 @@ from erniebot_agent.agents.callback.default import get_no_ellipsis_callback from erniebot_agent.agents.functional_agent import FunctionalAgent from erniebot_agent.chat_models.erniebot import ERNIEBot -from erniebot_agent.file import get_global_file_manager -from erniebot_agent.memory.sliding_window_memory import SlidingWindowMemory +from erniebot_agent.file import GlobalFileManager from erniebot_agent.memory import AIMessage, HumanMessage, Message +from erniebot_agent.memory.sliding_window_memory import SlidingWindowMemory from erniebot_agent.tools.base import Tool from erniebot_agent.tools.calculator_tool import CalculatorTool from erniebot_agent.tools.schema import ToolParameterView @@ -32,7 +32,7 @@ async def __call__(self, input_file_id: str, repeat_times: int) -> Dict[str, Any if "" in input_file_id: input_file_id = input_file_id.split("")[0] - file_manager = await get_global_file_manager() + file_manager = await GlobalFileManager().get() input_file = file_manager.look_up_file_by_id(input_file_id) if input_file is None: raise RuntimeError("File not found") @@ -121,8 +121,8 @@ def examples(self) -> List[Message]: async def run_agent(): - file_manager = await get_global_file_manager() - + file_manager = await GlobalFileManager().get() + docx_file = await file_manager.create_file_from_path( file_path="浅谈牛奶的营养与消费趋势.docx", file_type="remote", diff --git a/erniebot-agent/src/erniebot_agent/agents/base.py b/erniebot-agent/src/erniebot_agent/agents/base.py index 623480b5d..e07a005b9 100644 --- a/erniebot-agent/src/erniebot_agent/agents/base.py +++ b/erniebot-agent/src/erniebot_agent/agents/base.py @@ -27,7 +27,7 @@ ToolResponse, ) from erniebot_agent.chat_models.base import ChatModel -from erniebot_agent.file import get_global_file_manager, protocol +from erniebot_agent.file import GlobalFileManager, protocol from erniebot_agent.file.base import File from erniebot_agent.file.file_manager import FileManager from erniebot_agent.memory import Message, SystemMessage @@ -187,7 +187,7 @@ async def _sniff_and_extract_files_from_args( async def _get_file_manager(self) -> FileManager: if self._file_manager is None: - file_manager = await get_global_file_manager() + file_manager = await GlobalFileManager().get() else: file_manager = self._file_manager return file_manager diff --git a/erniebot-agent/src/erniebot_agent/agents/functional_agent.py b/erniebot-agent/src/erniebot_agent/agents/functional_agent.py index 8061de99c..6085c67e1 100644 --- a/erniebot-agent/src/erniebot_agent/agents/functional_agent.py +++ b/erniebot-agent/src/erniebot_agent/agents/functional_agent.py @@ -113,7 +113,7 @@ async def _async_plan( messages=messages, functions=self._tool_manager.get_tool_schemas(), system=self.system_message.content if self.system_message is not None else None, - plugins=self.plugins, + plugins=self._plugins, ) output_message = llm_resp.message chat_history.append(output_message) diff --git a/erniebot-agent/src/erniebot_agent/file/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py index 9a17172c3..c8616a3a7 100644 --- a/erniebot-agent/src/erniebot_agent/file/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file/__init__.py @@ -12,7 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -from erniebot_agent.file.global_file_manager import ( - configure_global_file_manager, - get_global_file_manager, -) +from erniebot_agent.file.global_file_manager import GlobalFileManager diff --git a/erniebot-agent/src/erniebot_agent/file/global_file_manager.py b/erniebot-agent/src/erniebot_agent/file/global_file_manager.py index 4b17c834f..818bc5d4c 100644 --- a/erniebot-agent/src/erniebot_agent/file/global_file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/global_file_manager.py @@ -13,52 +13,66 @@ # limitations under the License. import asyncio -from typing import Any, Optional +from typing import Any, Optional, final import asyncio_atexit # type: ignore from erniebot_agent.file.file_manager import FileManager from erniebot_agent.file.remote_file import AIStudioFileClient from erniebot_agent.utils import config_from_environ as C +from erniebot_agent.utils.misc import Singleton -_global_file_manager: Optional[FileManager] = None -_lock = asyncio.Lock() +@final +class GlobalFileManager(metaclass=Singleton): + _file_manager: Optional[FileManager] -async def get_global_file_manager() -> FileManager: - global _global_file_manager - async with _lock: - if _global_file_manager is None: - _global_file_manager = await _create_default_file_manager(access_token=None, save_dir=None) - return _global_file_manager + def __init__(self) -> None: + super().__init__() + self._lock = asyncio.Lock() + self._file_manager = None + async def get(self) -> FileManager: + async with self._lock: + if self._file_manager is None: + self._file_manager = await self._create_default_file_manager( + access_token=None, save_dir=None + ) + return self._file_manager -async def configure_global_file_manager( - access_token: Optional[str] = None, save_dir: Optional[str] = None, **opts: Any -) -> None: - global _global_file_manager - async with _lock: - if _global_file_manager is not None: - raise RuntimeError( - "The global file manager can only be configured once before calling `get_global_file_manager`." + async def configure( + self, + access_token: Optional[str] = None, + save_dir: Optional[str] = None, + **opts: Any, + ) -> None: + async with self._lock: + if self._file_manager is not None: + raise RuntimeError( + "`GlobalFileManager.configure` can only be called once" + " and must be called before calling `GlobalFileManager.get`." + ) + self._file_manager = await self._create_default_file_manager( + access_token=access_token, save_dir=save_dir, **opts ) - _global_file_manager = await _create_default_file_manager(access_token=access_token, save_dir=save_dir, **opts) + async def _create_default_file_manager( + self, + access_token: Optional[str], + save_dir: Optional[str], + **opts: Any, + ) -> FileManager: + async def _close_file_manager(): + await file_manager.close() -async def _create_default_file_manager( - access_token: Optional[str], save_dir: Optional[str], **opts: Any -) -> FileManager: - async def _close_file_manager(): - await file_manager.close() - - if access_token is None: - access_token = C.get_global_access_token() - if save_dir is None: - save_dir = C.get_global_save_dir() - if access_token is not None: - remote_file_client = AIStudioFileClient(access_token=access_token) - else: - remote_file_client = None - file_manager = FileManager(remote_file_client, save_dir, **opts) - asyncio_atexit.register(_close_file_manager) - return file_manager + if access_token is None: + access_token = C.get_global_access_token() + if save_dir is None: + save_dir = C.get_global_save_dir() + if access_token is not None: + remote_file_client = AIStudioFileClient(access_token=access_token) + else: + remote_file_client = None + file_manager = FileManager(remote_file_client, save_dir, **opts) + asyncio_atexit.register(_close_file_manager) + return file_manager diff --git a/erniebot-agent/src/erniebot_agent/tools/remote_tool.py b/erniebot-agent/src/erniebot_agent/tools/remote_tool.py index 06b387ce6..4c625bc00 100644 --- a/erniebot-agent/src/erniebot_agent/tools/remote_tool.py +++ b/erniebot-agent/src/erniebot_agent/tools/remote_tool.py @@ -4,10 +4,10 @@ import json from copy import deepcopy from typing import Any, Dict, List, Optional, Type -from erniebot_agent.file import get_global_file_manager import requests +from erniebot_agent.file import GlobalFileManager from erniebot_agent.file.file_manager import FileManager from erniebot_agent.memory import Message from erniebot_agent.tools.base import BaseTool @@ -300,7 +300,7 @@ def __adhoc_post_process__(self, tool_response: dict) -> dict: async def _get_file_manager(self) -> FileManager: if self.file_manager is None: - file_manager = await get_global_file_manager() + file_manager = await GlobalFileManager().get() else: file_manager = self.file_manager return file_manager diff --git a/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py b/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py index 94f7e60c1..db765085f 100644 --- a/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py +++ b/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py @@ -43,7 +43,7 @@ class RemoteToolkit: info: EndpointInfo servers: List[Endpoint] paths: List[RemoteToolView] - file_manager: FileManager + file_manager: Optional[FileManager] component_schemas: dict[str, Type[ToolParameterView]] headers: dict diff --git a/erniebot-agent/src/erniebot_agent/utils/mixins.py b/erniebot-agent/src/erniebot_agent/utils/mixins.py index 6a1a409e8..4438bdf3e 100644 --- a/erniebot-agent/src/erniebot_agent/utils/mixins.py +++ b/erniebot-agent/src/erniebot_agent/utils/mixins.py @@ -17,7 +17,7 @@ import base64 import os import tempfile -from typing import TYPE_CHECKING, Any, List, Protocol +from typing import TYPE_CHECKING, Any, List, Optional, Protocol, cast from erniebot_agent.utils.common import get_file_type from erniebot_agent.utils.exceptions import ObjectClosedError @@ -30,9 +30,13 @@ class GradioMixin: - _file_manager: FileManager # make mypy happy + _file_manager: Optional[FileManager] # make mypy happy _tool_manager: ToolManager # make mypy happy + # make mypy happy + async def _get_file_manager(self) -> FileManager: + return cast(FileManager, None) + def launch_gradio_demo(self, **launch_kwargs: Any): # TODO: Unified optional dependencies management try: @@ -71,7 +75,8 @@ async def _chat(history): ): # If there is a file output in the last round, then we need to show it output_file_id = response.files[-1].file.id - output_file = self._file_manager.look_up_file_by_id(output_file_id) + file_manager = await self._get_file_manager() + output_file = file_manager.look_up_file_by_id(output_file_id) file_content = await output_file.read_contents() if get_file_type(response.files[-1].file.filename) == "image": # If it is a image, we can display it in the same chat page @@ -114,13 +119,14 @@ def _clear(): async def _upload(file, history): nonlocal _uploaded_file_cache + file_manager = await self._get_file_manager() for single_file in file: - upload_file = await self._file_manager.create_file_from_path(single_file.name) + upload_file = await file_manager.create_file_from_path(single_file.name) _uploaded_file_cache.append(upload_file) history = history + [((single_file.name,), None)] size = len(file) - output_lis = self._file_manager.list_registered_files() + output_lis = file_manager.list_registered_files() item = "" for i in range(len(output_lis) - size): item += f'
  • {str(output_lis[i]).strip("<>")}
    • ' @@ -165,8 +171,7 @@ def _messages_to_dicts(messages): ) with gr.Accordion("Files", open=False): - file_lis = self._file_manager.list_registered_files() - all_files = gr.HTML(value=file_lis, label="All input files") + all_files = gr.HTML(value=[], label="All input files") with gr.Accordion("Tools", open=False): attached_tools = self._tool_manager.get_tools() tool_descriptions = [tool.function_call_schema() for tool in attached_tools] diff --git a/erniebot-agent/test_concurrent.py b/erniebot-agent/test_concurrent.py new file mode 100644 index 000000000..1f74abcfb --- /dev/null +++ b/erniebot-agent/test_concurrent.py @@ -0,0 +1,10 @@ +import asyncio +from erniebot_agent.file import GlobalFileManager + +async def fun(): + await GlobalFileManager().get() + +async def main(): + await asyncio.gather(fun(), fun(), fun(), fun()) + +asyncio.run(main()) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py b/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py index d57373e86..501d20cca 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py @@ -1,8 +1,8 @@ from __future__ import annotations -from erniebot_agent.file.file_manager import FileManager import pytest +from erniebot_agent.file.file_manager import FileManager from erniebot_agent.tools import RemoteToolkit from .base import RemoteToolTesting From 29be29b0e6aa962ef10054c47094b16cd1d2696a Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Fri, 22 Dec 2023 17:55:07 +0800 Subject: [PATCH 21/63] Show file type --- erniebot-agent/src/erniebot_agent/agents/base.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/erniebot-agent/src/erniebot_agent/agents/base.py b/erniebot-agent/src/erniebot_agent/agents/base.py index e07a005b9..50387a841 100644 --- a/erniebot-agent/src/erniebot_agent/agents/base.py +++ b/erniebot-agent/src/erniebot_agent/agents/base.py @@ -176,7 +176,10 @@ async def _sniff_and_extract_files_from_args( file_manager = await self._get_file_manager() file = file_manager.look_up_file_by_id(val) if file is None: - raise RuntimeError(f"Unregistered file with ID {repr(val)} is used by {repr(tool)}.") + raise RuntimeError( + f"Unregistered file with ID {repr(val)} is used by {repr(tool)}." + f" File type: {file_type}" + ) agent_files.append(AgentFile(file=file, type=file_type, used_by=tool.tool_name)) elif isinstance(val, dict): agent_files.extend(await self._sniff_and_extract_files_from_args(val, tool, file_type)) From b74ead7c96ea92eae52e44633b384c8ea078b8a5 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Fri, 22 Dec 2023 18:57:08 +0800 Subject: [PATCH 22/63] Fix bugs --- erniebot-agent/examples/cv_agent/CV_agent.py | 6 +++--- .../examples/plugins/multiple_plugins.py | 6 +++--- erniebot-agent/src/erniebot_agent/agents/base.py | 14 ++++++++------ erniebot-agent/src/erniebot_agent/file/__init__.py | 2 +- .../src/erniebot_agent/file/file_manager.py | 2 +- ...e_manager.py => global_file_manager_handler.py} | 6 +++--- .../src/erniebot_agent/tools/remote_tool.py | 4 ++-- erniebot-agent/src/erniebot_agent/utils/mixins.py | 2 ++ erniebot-agent/test_concurrent.py | 4 ++-- 9 files changed, 25 insertions(+), 21 deletions(-) rename erniebot-agent/src/erniebot_agent/file/{global_file_manager.py => global_file_manager_handler.py} (94%) diff --git a/erniebot-agent/examples/cv_agent/CV_agent.py b/erniebot-agent/examples/cv_agent/CV_agent.py index b79281900..6110a7047 100644 --- a/erniebot-agent/examples/cv_agent/CV_agent.py +++ b/erniebot-agent/examples/cv_agent/CV_agent.py @@ -2,7 +2,7 @@ from erniebot_agent.agents.functional_agent import FunctionalAgent from erniebot_agent.chat_models.erniebot import ERNIEBot -from erniebot_agent.file import GlobalFileManager +from erniebot_agent.file import GlobalFileManagerHandler from erniebot_agent.memory.whole_memory import WholeMemory from erniebot_agent.tools import RemoteToolkit @@ -15,14 +15,14 @@ def __init__(self): async def run_agent(): - await GlobalFileManager().configure(access_token="") + await GlobalFileManagerHandler().configure(access_token="") llm = ERNIEBot(model="ernie-bot", api_type="aistudio", access_token="") toolkit = CVToolkit() memory = WholeMemory() agent = FunctionalAgent(llm=llm, tools=toolkit.tools, memory=memory) - file_manager = await GlobalFileManager().get() + file_manager = await GlobalFileManagerHandler().get() seg_file = await file_manager.create_file_from_path(file_path="cityscapes_demo.png", file_type="local") clas_file = await file_manager.create_file_from_path(file_path="class_img.jpg", file_type="local") ocr_file = await file_manager.create_file_from_path(file_path="ch.png", file_type="local") diff --git a/erniebot-agent/examples/plugins/multiple_plugins.py b/erniebot-agent/examples/plugins/multiple_plugins.py index 411a9f265..a2b3fcdfe 100644 --- a/erniebot-agent/examples/plugins/multiple_plugins.py +++ b/erniebot-agent/examples/plugins/multiple_plugins.py @@ -6,7 +6,7 @@ from erniebot_agent.agents.callback.default import get_no_ellipsis_callback from erniebot_agent.agents.functional_agent import FunctionalAgent from erniebot_agent.chat_models.erniebot import ERNIEBot -from erniebot_agent.file import GlobalFileManager +from erniebot_agent.file import GlobalFileManagerHandler from erniebot_agent.memory import AIMessage, HumanMessage, Message from erniebot_agent.memory.sliding_window_memory import SlidingWindowMemory from erniebot_agent.tools.base import Tool @@ -32,7 +32,7 @@ async def __call__(self, input_file_id: str, repeat_times: int) -> Dict[str, Any if "" in input_file_id: input_file_id = input_file_id.split("")[0] - file_manager = await GlobalFileManager().get() + file_manager = await GlobalFileManagerHandler().get() input_file = file_manager.look_up_file_by_id(input_file_id) if input_file is None: raise RuntimeError("File not found") @@ -121,7 +121,7 @@ def examples(self) -> List[Message]: async def run_agent(): - file_manager = await GlobalFileManager().get() + file_manager = await GlobalFileManagerHandler().get() docx_file = await file_manager.create_file_from_path( file_path="浅谈牛奶的营养与消费趋势.docx", diff --git a/erniebot-agent/src/erniebot_agent/agents/base.py b/erniebot-agent/src/erniebot_agent/agents/base.py index 50387a841..5e96ab545 100644 --- a/erniebot-agent/src/erniebot_agent/agents/base.py +++ b/erniebot-agent/src/erniebot_agent/agents/base.py @@ -27,13 +27,14 @@ ToolResponse, ) from erniebot_agent.chat_models.base import ChatModel -from erniebot_agent.file import GlobalFileManager, protocol +from erniebot_agent.file import GlobalFileManagerHandler, protocol from erniebot_agent.file.base import File from erniebot_agent.file.file_manager import FileManager from erniebot_agent.memory import Message, SystemMessage from erniebot_agent.memory.base import Memory from erniebot_agent.tools.base import BaseTool from erniebot_agent.tools.tool_manager import ToolManager +from erniebot_agent.utils.exceptions import FileError from erniebot_agent.utils.mixins import GradioMixin logger = logging.getLogger(__name__) @@ -174,12 +175,13 @@ async def _sniff_and_extract_files_from_args( if isinstance(val, str): if protocol.is_file_id(val): file_manager = await self._get_file_manager() - file = file_manager.look_up_file_by_id(val) - if file is None: - raise RuntimeError( + try: + file = file_manager.look_up_file_by_id(val) + except FileError as e: + raise FileError( f"Unregistered file with ID {repr(val)} is used by {repr(tool)}." f" File type: {file_type}" - ) + ) from e agent_files.append(AgentFile(file=file, type=file_type, used_by=tool.tool_name)) elif isinstance(val, dict): agent_files.extend(await self._sniff_and_extract_files_from_args(val, tool, file_type)) @@ -190,7 +192,7 @@ async def _sniff_and_extract_files_from_args( async def _get_file_manager(self) -> FileManager: if self._file_manager is None: - file_manager = await GlobalFileManager().get() + file_manager = await GlobalFileManagerHandler().get() else: file_manager = self._file_manager return file_manager diff --git a/erniebot-agent/src/erniebot_agent/file/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py index c8616a3a7..2d89f4142 100644 --- a/erniebot-agent/src/erniebot_agent/file/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file/__init__.py @@ -12,4 +12,4 @@ # See the License for the specific language governing permissions and # limitations under the License. -from erniebot_agent.file.global_file_manager import GlobalFileManager +from erniebot_agent.file.global_file_manager_handler import GlobalFileManagerHandler diff --git a/erniebot-agent/src/erniebot_agent/file/file_manager.py b/erniebot-agent/src/erniebot_agent/file/file_manager.py index 8945d23b6..26f08a998 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/file_manager.py @@ -257,7 +257,7 @@ async def list_remote_files(self) -> List[RemoteFile]: files = await self.remote_file_client.list_files() return files - def look_up_file_by_id(self, file_id: str) -> Optional[File]: + def look_up_file_by_id(self, file_id: str) -> File: self.ensure_not_closed() file = self._file_registry.look_up_file(file_id) if file is None: diff --git a/erniebot-agent/src/erniebot_agent/file/global_file_manager.py b/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py similarity index 94% rename from erniebot-agent/src/erniebot_agent/file/global_file_manager.py rename to erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py index 818bc5d4c..4d368179d 100644 --- a/erniebot-agent/src/erniebot_agent/file/global_file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py @@ -24,7 +24,7 @@ @final -class GlobalFileManager(metaclass=Singleton): +class GlobalFileManagerHandler(metaclass=Singleton): _file_manager: Optional[FileManager] def __init__(self) -> None: @@ -49,8 +49,8 @@ async def configure( async with self._lock: if self._file_manager is not None: raise RuntimeError( - "`GlobalFileManager.configure` can only be called once" - " and must be called before calling `GlobalFileManager.get`." + "`GlobalFileManagerHandler.configure` can only be called once" + " and must be called before calling `GlobalFileManagerHandler.get`." ) self._file_manager = await self._create_default_file_manager( access_token=access_token, save_dir=save_dir, **opts diff --git a/erniebot-agent/src/erniebot_agent/tools/remote_tool.py b/erniebot-agent/src/erniebot_agent/tools/remote_tool.py index 4c625bc00..4edf1baee 100644 --- a/erniebot-agent/src/erniebot_agent/tools/remote_tool.py +++ b/erniebot-agent/src/erniebot_agent/tools/remote_tool.py @@ -7,7 +7,7 @@ import requests -from erniebot_agent.file import GlobalFileManager +from erniebot_agent.file import GlobalFileManagerHandler from erniebot_agent.file.file_manager import FileManager from erniebot_agent.memory import Message from erniebot_agent.tools.base import BaseTool @@ -300,7 +300,7 @@ def __adhoc_post_process__(self, tool_response: dict) -> dict: async def _get_file_manager(self) -> FileManager: if self.file_manager is None: - file_manager = await GlobalFileManager().get() + file_manager = await GlobalFileManagerHandler().get() else: file_manager = self.file_manager return file_manager diff --git a/erniebot-agent/src/erniebot_agent/utils/mixins.py b/erniebot-agent/src/erniebot_agent/utils/mixins.py index 4438bdf3e..21fe4f335 100644 --- a/erniebot-agent/src/erniebot_agent/utils/mixins.py +++ b/erniebot-agent/src/erniebot_agent/utils/mixins.py @@ -38,6 +38,8 @@ async def _get_file_manager(self) -> FileManager: return cast(FileManager, None) def launch_gradio_demo(self, **launch_kwargs: Any): + # XXX: The current implementation requires that the inheriting objects + # be constructed outside an event loop, which is probably not sensible. # TODO: Unified optional dependencies management try: import gradio as gr # type: ignore diff --git a/erniebot-agent/test_concurrent.py b/erniebot-agent/test_concurrent.py index 1f74abcfb..e0f31764a 100644 --- a/erniebot-agent/test_concurrent.py +++ b/erniebot-agent/test_concurrent.py @@ -1,8 +1,8 @@ import asyncio -from erniebot_agent.file import GlobalFileManager +from erniebot_agent.file import GlobalFileManagerHandler async def fun(): - await GlobalFileManager().get() + await GlobalFileManagerHandler().get() async def main(): await asyncio.gather(fun(), fun(), fun(), fun()) From 9b732203da4d1fccf74d1fdc72ca563f134512ec Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Fri, 22 Dec 2023 19:00:57 +0800 Subject: [PATCH 23/63] Fix linting issues --- .../src/erniebot_agent/agents/base.py | 1 - .../erniebot_agent/agents/functional_agent.py | 7 +- .../agents/functional_agent_with_retrieval.py | 2 +- .../src/erniebot_agent/memory/__init__.py | 9 ++- .../src/erniebot_agent/memory/base.py | 6 +- .../memory/limit_tokens_memory.py | 12 +-- .../src/erniebot_agent/memory/messages.py | 77 ++++++++++--------- .../memory/sliding_window_memory.py | 2 +- .../src/erniebot_agent/memory/whole_memory.py | 1 - 9 files changed, 65 insertions(+), 52 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/agents/base.py b/erniebot-agent/src/erniebot_agent/agents/base.py index 2441b4f6f..b7bde52b5 100644 --- a/erniebot-agent/src/erniebot_agent/agents/base.py +++ b/erniebot-agent/src/erniebot_agent/agents/base.py @@ -30,7 +30,6 @@ from erniebot_agent.file import GlobalFileManagerHandler, protocol from erniebot_agent.file.base import File from erniebot_agent.file.file_manager import FileManager -from erniebot_agent.file.protocol import is_local_file_id, is_remote_file_id from erniebot_agent.memory import Memory from erniebot_agent.memory.messages import Message, SystemMessage from erniebot_agent.tools.base import BaseTool diff --git a/erniebot-agent/src/erniebot_agent/agents/functional_agent.py b/erniebot-agent/src/erniebot_agent/agents/functional_agent.py index 884045bcd..89b6f5be2 100644 --- a/erniebot-agent/src/erniebot_agent/agents/functional_agent.py +++ b/erniebot-agent/src/erniebot_agent/agents/functional_agent.py @@ -21,8 +21,13 @@ from erniebot_agent.chat_models.base import ChatModel from erniebot_agent.file.base import File from erniebot_agent.file.file_manager import FileManager -from erniebot_agent.memory.messages import FunctionMessage, HumanMessage, Message, SystemMessage from erniebot_agent.memory import Memory +from erniebot_agent.memory.messages import ( + FunctionMessage, + HumanMessage, + Message, + SystemMessage, +) from erniebot_agent.tools.base import BaseTool _MAX_STEPS = 5 diff --git a/erniebot-agent/src/erniebot_agent/agents/functional_agent_with_retrieval.py b/erniebot-agent/src/erniebot_agent/agents/functional_agent_with_retrieval.py index 092de7ce7..6f9765c36 100644 --- a/erniebot-agent/src/erniebot_agent/agents/functional_agent_with_retrieval.py +++ b/erniebot-agent/src/erniebot_agent/agents/functional_agent_with_retrieval.py @@ -18,8 +18,8 @@ FunctionMessage, HumanMessage, Message, + SearchInfo, ) -from erniebot_agent.memory.messages import SearchInfo from erniebot_agent.prompt import PromptTemplate from erniebot_agent.retrieval import BaizhongSearch from erniebot_agent.tools.base import Tool diff --git a/erniebot-agent/src/erniebot_agent/memory/__init__.py b/erniebot-agent/src/erniebot_agent/memory/__init__.py index 981cee090..4f7b2f76b 100644 --- a/erniebot-agent/src/erniebot_agent/memory/__init__.py +++ b/erniebot-agent/src/erniebot_agent/memory/__init__.py @@ -14,6 +14,13 @@ from .base import Memory from .limit_tokens_memory import LimitTokensMemory -from .messages import AIMessage, FunctionMessage, HumanMessage, Message, SystemMessage, AIMessageChunk +from .messages import ( + AIMessage, + AIMessageChunk, + FunctionMessage, + HumanMessage, + Message, + SystemMessage, +) from .sliding_window_memory import SlidingWindowMemory from .whole_memory import WholeMemory diff --git a/erniebot-agent/src/erniebot_agent/memory/base.py b/erniebot-agent/src/erniebot_agent/memory/base.py index 1913d4aa0..604fcd239 100644 --- a/erniebot-agent/src/erniebot_agent/memory/base.py +++ b/erniebot-agent/src/erniebot_agent/memory/base.py @@ -21,7 +21,7 @@ class MessageManager: """ Messages Manager, manage the messages of a conversation. - + Attributes: messages (List[Message]): the messages of a conversation. system_message (SystemMessage): the system message of a conversation. @@ -73,10 +73,10 @@ def retrieve_messages(self) -> List[Message]: class Memory: """ The base class of memory - + Attributes: msg_manager (MessageManager): the message manager of a conversation. - + Returns: A memory object. """ diff --git a/erniebot-agent/src/erniebot_agent/memory/limit_tokens_memory.py b/erniebot-agent/src/erniebot_agent/memory/limit_tokens_memory.py index 5a6b3b45c..e8e9d1498 100644 --- a/erniebot-agent/src/erniebot_agent/memory/limit_tokens_memory.py +++ b/erniebot-agent/src/erniebot_agent/memory/limit_tokens_memory.py @@ -23,18 +23,18 @@ class LimitTokensMemory(Memory): Args: max_token_limit (int): The maximum number of tokens in the context. - + Attributes: max_token_limit (int): The maximum number of tokens in the context. mem_token_count (int): The number of tokens in the context. - + Examples: .. code-block:: python from erniebot_agent.memory import LimitTokensMemory memory = LimitTokensMemory(max_token_limit=3000) memory.add_message(AIMessage("Hello world!")) - + """ def __init__(self, max_token_limit=3000): @@ -52,10 +52,10 @@ def __init__(self, max_token_limit=3000): def add_message(self, message: Message): """ Add a message to memory. Prune the message if number of tokens in memory >= max_token_limit. - + Args: message (Message): The message to be added. - + Returns: None """ @@ -68,7 +68,7 @@ def add_message(self, message: Message): def prune_message(self): """ Prune the message if number of tokens in memory >= max_token_limit. - + Raises: RuntimeError: If the message is empty after pruning. diff --git a/erniebot-agent/src/erniebot_agent/memory/messages.py b/erniebot-agent/src/erniebot_agent/memory/messages.py index 0e409527b..3f999db90 100644 --- a/erniebot-agent/src/erniebot_agent/memory/messages.py +++ b/erniebot-agent/src/erniebot_agent/memory/messages.py @@ -46,12 +46,12 @@ class Message: role (str): character of the message. content (str): content of the message. token_count (Optional[int], optional): number of tokens of the message content. Defaults to None. - + Attributes: - role (str): character of the message. - content (str): content of the message. - token_count (Optional[int]): number of tokens of the message content. - + role (str): character of the message. + content (str): content of the message. + token_count (Optional[int]): number of tokens of the message content. + Examples: >>> Message("user", "hello") @@ -59,6 +59,7 @@ class Message: """ + def __init__(self, role: str, content: str, token_count: Optional[int] = None): self._role = role self._content = content @@ -117,12 +118,12 @@ class SystemMessage(Message): Args: content (str): the content of the message. - + Attributes: - role (str): character of the message. - content (str): content of the message. - token_count (Optional[int]): number of tokens of the message content. - + role (str): character of the message. + content (str): content of the message. + token_count (Optional[int]): number of tokens of the message content. + Examples: .. code-block:: python @@ -138,6 +139,7 @@ class SystemMessage(Message): 3 """ + def __init__(self, content: str): super().__init__(role="system", content=content, token_count=len(content)) @@ -145,15 +147,15 @@ def __init__(self, content: str): class HumanMessage(Message): """ The definition of the message created by a human. - + Args: content (str): the content of the message. - + Attributes: - role (str): character of the message. - content (str): content of the message. - token_count (Optional[int]): number of tokens of the message content. - + role (str): character of the message. + content (str): content of the message. + token_count (Optional[int]): number of tokens of the message content. + Examples: .. code-block:: python >>> from erniebot_agent.messages import HumanMessage @@ -167,7 +169,7 @@ class HumanMessage(Message): prompt, files, include_file_urls=True) >>> message File-local-xxxx{url}.> - + """ def __init__(self, content: str): @@ -179,18 +181,19 @@ async def create_with_files( ) -> Self: """ create a Human Message with file input - + Args: text: content of the message. - files (List[File]): The file that the message contains. + files (List[File]): The file that the message contains. include_file_urls: Whehter to include file URLs in the content of message. - + Returns: A HumanMessage object that contains file in the content. - + Raises: RuntimeError: Only `RemoteFile` objects can set include_file_urls as True. """ + def _get_file_reprs(files: List[File]) -> List[str]: file_reprs = [] for file in files: @@ -233,25 +236,25 @@ class AIMessage(Message): content (str): the content of the message. function_call (Optional[FunctionCall], optional): The function that agent calls. Defaults to None. token_usage (Optional[TokenUsage], optional): the token usage calculate by ERNIE. Defaults to None. - search_info (Optional[SearchInfo], optional): + search_info (Optional[SearchInfo], optional): The SearchInfo content of the chat model's response. Defaults to None. - + Attributes: - role (str): character of the message. - content (str): content of the message. - token_count (Optional[int]): number of tokens of the message content. + role (str): character of the message. + content (str): content of the message. + token_count (Optional[int]): number of tokens of the message content. function_call (Optional[FunctionCall]): The function that agent calls. query_tokens_count (int): the number of tokens in the query. search_info (Optional[SearchInfo]): The SearchInfo in the chat model's response. - + Examples: - + .. code-block:: python >>> human_message = HumanMessage(content="What is the text in this image?") >>> ai_message = AIMessage( - function_call={"name": "OCR", "thoughts": "The user want to know the text in the image, - I need to use the OCR tool", + function_call={"name": "OCR", "thoughts": "The user want to know the text in the image, + I need to use the OCR tool", "arguments": "{\"imgae_byte_str\": file-remote-xxxx, \"lang\": "en"}"}, token_usage={"prompt_tokens": 10, "completion_tokens": 20}, search_info={}]} @@ -290,17 +293,17 @@ def _parse_token_count(self, token_usage: TokenUsage): class FunctionMessage(Message): """ The definition of a message that calls tools, containing the result of a function call. - + Args: name (str): the name of the function. content (str): the content of the message. - + Attributes: - name (str): the name of the function. - role (str): character of the message. - content (str): content of the message. - token_count (Optional[int]): number of tokens of the message content. - + name (str): the name of the function. + role (str): character of the message. + content (str): content of the message. + token_count (Optional[int]): number of tokens of the message content. + Examples: .. code-block:: python diff --git a/erniebot-agent/src/erniebot_agent/memory/sliding_window_memory.py b/erniebot-agent/src/erniebot_agent/memory/sliding_window_memory.py index 69699390b..627e29c43 100644 --- a/erniebot-agent/src/erniebot_agent/memory/sliding_window_memory.py +++ b/erniebot-agent/src/erniebot_agent/memory/sliding_window_memory.py @@ -22,7 +22,7 @@ class SlidingWindowMemory(Memory): Each round contains a piece of human message and a piece of AI message. Attributes: - max_round(int): Max number of rounds. + max_round(int): Max number of rounds. retained_round(int): The first number of rounds of memory will be preserverd. Default to 0. Raises: diff --git a/erniebot-agent/src/erniebot_agent/memory/whole_memory.py b/erniebot-agent/src/erniebot_agent/memory/whole_memory.py index 28d42455a..73d3f5235 100644 --- a/erniebot-agent/src/erniebot_agent/memory/whole_memory.py +++ b/erniebot-agent/src/erniebot_agent/memory/whole_memory.py @@ -25,4 +25,3 @@ class WholeMemory(Memory): def __init__(self): super().__init__() - From b19b42ad17ec3cb87e489ac249f7da4b07bd8ca7 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Fri, 22 Dec 2023 19:01:30 +0800 Subject: [PATCH 24/63] Fix linting issues --- erniebot-agent/src/erniebot_agent/agents/schema.py | 1 - 1 file changed, 1 deletion(-) diff --git a/erniebot-agent/src/erniebot_agent/agents/schema.py b/erniebot-agent/src/erniebot_agent/agents/schema.py index 858e945cf..21228756b 100644 --- a/erniebot-agent/src/erniebot_agent/agents/schema.py +++ b/erniebot-agent/src/erniebot_agent/agents/schema.py @@ -20,7 +20,6 @@ from erniebot_agent.file import protocol from erniebot_agent.file.base import File -from erniebot_agent.file.protocol import extract_file_ids from erniebot_agent.memory.messages import AIMessage, Message From 1f281649a878247d7066232b597eea41244b32ab Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Fri, 22 Dec 2023 19:26:52 +0800 Subject: [PATCH 25/63] Fix integration tests --- .../src/erniebot_agent/memory/messages.py | 1 - .../tests/integration_tests/apihub/base.py | 5 +++-- .../tests/integration_tests/apihub/test_car_cls.py | 2 +- .../tests/integration_tests/apihub/test_dish.py | 2 +- .../integration_tests/apihub/test_doc_analysis.py | 4 ++-- .../integration_tests/apihub/test_doc_rm_bnd.py | 2 +- .../integration_tests/apihub/test_handwriting.py | 2 +- .../integration_tests/apihub/test_high_ocr.py | 2 +- .../integration_tests/apihub/test_image2text.py | 2 +- .../apihub/test_image_moderation.py | 2 +- .../integration_tests/apihub/test_img_enhance.py | 2 +- .../integration_tests/apihub/test_img_transform.py | 4 ++-- .../integration_tests/apihub/test_landmark.py | 2 +- .../tests/integration_tests/apihub/test_ocr.py | 10 +++++----- .../integration_tests/apihub/test_pic_translate.py | 2 +- .../integration_tests/apihub/test_pp_models.py | 14 +++++++------- .../integration_tests/apihub/test_pp_shituv2.py | 2 +- .../apihub/test_text_moderation.py | 2 +- .../apihub/test_text_to_speech.py | 2 +- .../integration_tests/apihub/test_translation.py | 2 +- .../mocks/mock_remote_file_client_server.py | 4 ++-- 21 files changed, 35 insertions(+), 35 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/memory/messages.py b/erniebot-agent/src/erniebot_agent/memory/messages.py index 3f999db90..b173bb0e4 100644 --- a/erniebot-agent/src/erniebot_agent/memory/messages.py +++ b/erniebot-agent/src/erniebot_agent/memory/messages.py @@ -162,7 +162,6 @@ class HumanMessage(Message): >>> HumanMessage("I want to order a pizza.") - >>> from erniebot_agent.file_io.base import File >>> prompt = "What is the text in this image?" >>> files = [await file_manager.create_file_from_path(file_path="ocr_img.jpg", file_type="remote")] >>> message = await HumanMessage.create_with_files( diff --git a/erniebot-agent/tests/integration_tests/apihub/base.py b/erniebot-agent/tests/integration_tests/apihub/base.py index 1a8c92812..84f911549 100644 --- a/erniebot-agent/tests/integration_tests/apihub/base.py +++ b/erniebot-agent/tests/integration_tests/apihub/base.py @@ -10,7 +10,7 @@ from erniebot_agent.agents.functional_agent import FunctionalAgent from erniebot_agent.chat_models import ERNIEBot -from erniebot_agent.file_io.file_manager import FileManager +from erniebot_agent.file.file_manager import FileManager from erniebot_agent.memory import WholeMemory from erniebot_agent.tools import RemoteToolkit from erniebot_agent.tools.tool_manager import ToolManager @@ -44,10 +44,11 @@ def get_agent(self, toolkit: RemoteToolkit): if "EB_BASE_URL" in os.environ: llm = ERNIEBot(model="ernie-3.5", api_type="custom") else: - llm = ERNIEBot(model="ernie-3.5") + llm = ERNIEBot(model="ernie-3.5", api_type="aistudio") return FunctionalAgent( llm=llm, tools=ToolManager(tools=toolkit.get_tools()), memory=WholeMemory(), + file_manager=self.file_manager, ) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_car_cls.py b/erniebot-agent/tests/integration_tests/apihub/test_car_cls.py index d158ff741..89854a1d2 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_car_cls.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_car_cls.py @@ -10,7 +10,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_car_classify(self): - toolkit = RemoteToolkit.from_aistudio("car-classify") + toolkit = RemoteToolkit.from_aistudio("car-classify", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path(self.download_fixture_file("biyadi.png")) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_dish.py b/erniebot-agent/tests/integration_tests/apihub/test_dish.py index e49a5c226..81794bff8 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_dish.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_dish.py @@ -10,7 +10,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_dish_classify(self): - toolkit = RemoteToolkit.from_aistudio("dish-classify") + toolkit = RemoteToolkit.from_aistudio("dish-classify", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path(self.download_fixture_file("dish.png")) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_doc_analysis.py b/erniebot-agent/tests/integration_tests/apihub/test_doc_analysis.py index b99342a3b..7ed729f9d 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_doc_analysis.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_doc_analysis.py @@ -14,7 +14,7 @@ async def asyncSetUp(self) -> None: @pytest.mark.asyncio async def test_doc_analysis(self): - toolkit = RemoteToolkit.from_aistudio("doc-analysis") + toolkit = RemoteToolkit.from_aistudio("doc-analysis", file_manager=self.file_manager) agent = self.get_agent(toolkit) @@ -23,7 +23,7 @@ async def test_doc_analysis(self): self.assertIn("城市管理执法办法", result.text) async def test_official_doc(self): - toolkit = RemoteToolkit.from_aistudio("official-doc-rec") + toolkit = RemoteToolkit.from_aistudio("official-doc-rec", file_manager=self.file_manager) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_doc_rm_bnd.py b/erniebot-agent/tests/integration_tests/apihub/test_doc_rm_bnd.py index 64bdea0a5..65185e8c4 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_doc_rm_bnd.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_doc_rm_bnd.py @@ -10,7 +10,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_doc_rm_bnd(self): - toolkit = RemoteToolkit.from_aistudio("rm-doc-img-bnd") + toolkit = RemoteToolkit.from_aistudio("rm-doc-img-bnd", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path(self.download_fixture_file("biyadi.png")) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_handwriting.py b/erniebot-agent/tests/integration_tests/apihub/test_handwriting.py index afdf6fa1c..cd276a15b 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_handwriting.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_handwriting.py @@ -16,7 +16,7 @@ async def asyncSetUp(self) -> None: @pytest.mark.asyncio async def test_hand_text_rec(self): - toolkit = RemoteToolkit.from_aistudio("hand-text-rec") + toolkit = RemoteToolkit.from_aistudio("hand-text-rec", file_manager=self.file_manager) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_high_ocr.py b/erniebot-agent/tests/integration_tests/apihub/test_high_ocr.py index cf13d3520..ef2982d39 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_high_ocr.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_high_ocr.py @@ -10,7 +10,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_tool(self): - toolkit = RemoteToolkit.from_aistudio("highacc-ocr") + toolkit = RemoteToolkit.from_aistudio("highacc-ocr", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path(self.download_fixture_file("shouxiezi.png")) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_image2text.py b/erniebot-agent/tests/integration_tests/apihub/test_image2text.py index f859d64c8..6adbb26c8 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_image2text.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_image2text.py @@ -10,7 +10,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_tool(self): - toolkit = RemoteToolkit.from_aistudio("image2text") + toolkit = RemoteToolkit.from_aistudio("image2text", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path(self.download_fixture_file("shouxiezi.png")) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_image_moderation.py b/erniebot-agent/tests/integration_tests/apihub/test_image_moderation.py index ed2ce5bcb..09effd55e 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_image_moderation.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_image_moderation.py @@ -10,7 +10,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_tool(self): - toolkit = RemoteToolkit.from_aistudio("image-moderation") + toolkit = RemoteToolkit.from_aistudio("image-moderation", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path(self.download_fixture_file("shouxiezi.png")) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_img_enhance.py b/erniebot-agent/tests/integration_tests/apihub/test_img_enhance.py index 7744155b6..152be3af6 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_img_enhance.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_img_enhance.py @@ -10,7 +10,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_tool(self): - toolkit = RemoteToolkit.from_aistudio("img-enhance") + toolkit = RemoteToolkit.from_aistudio("img-enhance", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path(self.download_fixture_file("shouxiezi.png")) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_img_transform.py b/erniebot-agent/tests/integration_tests/apihub/test_img_transform.py index 2d3cc338d..749f3f159 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_img_transform.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_img_transform.py @@ -14,7 +14,7 @@ async def asyncSetUp(self) -> None: @pytest.mark.asyncio async def test_img_style_trans(self): - toolkit = RemoteToolkit.from_aistudio("img-style-trans") + toolkit = RemoteToolkit.from_aistudio("img-style-trans", file_manager=self.file_manager) agent = self.get_agent(toolkit) @@ -23,7 +23,7 @@ async def test_img_style_trans(self): @pytest.mark.asyncio async def test_person_animation(self): - toolkit = RemoteToolkit.from_aistudio("person-animation") + toolkit = RemoteToolkit.from_aistudio("person-animation", file_manager=self.file_manager) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_landmark.py b/erniebot-agent/tests/integration_tests/apihub/test_landmark.py index c22034e84..70a283afc 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_landmark.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_landmark.py @@ -10,7 +10,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_dish_classify(self): - toolkit = RemoteToolkit.from_aistudio("landmark-rec") + toolkit = RemoteToolkit.from_aistudio("landmark-rec", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path( self.download_fixture_file("shanghai-dongfangmingzhu.png") diff --git a/erniebot-agent/tests/integration_tests/apihub/test_ocr.py b/erniebot-agent/tests/integration_tests/apihub/test_ocr.py index 3199a1164..9cd563687 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_ocr.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_ocr.py @@ -18,7 +18,7 @@ async def asyncSetUp(self) -> None: @pytest.mark.asyncio async def test_ocr_general(self): - toolkit = RemoteToolkit.from_aistudio("ocr-general") + toolkit = RemoteToolkit.from_aistudio("ocr-general", file_manager=self.file_manager) agent = self.get_agent(toolkit) @@ -28,7 +28,7 @@ async def test_ocr_general(self): @pytest.mark.asyncio async def test_ocr_pp(self): - toolkit = RemoteToolkit.from_aistudio("pp-structure-v2") + toolkit = RemoteToolkit.from_aistudio("pp-structure-v2", file_manager=self.file_manager) agent = self.get_agent(toolkit) @@ -38,7 +38,7 @@ async def test_ocr_pp(self): @pytest.mark.asyncio async def test_pp_ocr_v4(self): - toolkit = RemoteToolkit.from_aistudio("pp-ocrv4") + toolkit = RemoteToolkit.from_aistudio("pp-ocrv4", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path( self.download_fixture_file("ocr_example_input.png") @@ -53,7 +53,7 @@ async def test_pp_ocr_v4(self): @pytest.mark.asyncio async def test_shopping_receipt(self): - toolkit = RemoteToolkit.from_aistudio("shopping-receipt") + toolkit = RemoteToolkit.from_aistudio("shopping-receipt", file_manager=self.file_manager) agent = self.get_agent(toolkit) file = await self.file_manager.create_file_from_path(self.download_fixture_file("xiaopiao.png")) @@ -63,7 +63,7 @@ async def test_shopping_receipt(self): @pytest.mark.asyncio async def test_formula(self): - toolkit = RemoteToolkit.from_aistudio("formula") + toolkit = RemoteToolkit.from_aistudio("formula", file_manager=self.file_manager) agent = self.get_agent(toolkit) file = await self.file_manager.create_file_from_path(self.download_fixture_file("fomula.png")) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_pic_translate.py b/erniebot-agent/tests/integration_tests/apihub/test_pic_translate.py index 90a4b14a4..8974c5030 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_pic_translate.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_pic_translate.py @@ -10,7 +10,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_tool(self): - toolkit = RemoteToolkit.from_aistudio("pic-translate") + toolkit = RemoteToolkit.from_aistudio("pic-translate", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path(self.download_fixture_file("shouxiezi.png")) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_pp_models.py b/erniebot-agent/tests/integration_tests/apihub/test_pp_models.py index 89daf0246..f177ec7e2 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_pp_models.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_pp_models.py @@ -12,7 +12,7 @@ class TestPPRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_pp_matting(self): - toolkit = RemoteToolkit.from_aistudio("pp-matting") + toolkit = RemoteToolkit.from_aistudio("pp-matting", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path(self.download_fixture_file("trans.png")) agent = self.get_agent(toolkit) @@ -23,7 +23,7 @@ async def test_pp_matting(self): @pytest.mark.asyncio async def test_pp_human_v2(self): - toolkit = RemoteToolkit.from_aistudio("pp-human-v2") + toolkit = RemoteToolkit.from_aistudio("pp-human-v2", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path(self.download_fixture_file("human_attr.jpg")) agent = self.get_agent(toolkit) @@ -33,7 +33,7 @@ async def test_pp_human_v2(self): @pytest.mark.asyncio async def test_pp_humansegv2(self): - toolkit = RemoteToolkit.from_aistudio("humanseg") + toolkit = RemoteToolkit.from_aistudio("humanseg", file_manager=self.file_manager) agent = self.get_agent(toolkit) @@ -46,7 +46,7 @@ async def test_pp_humansegv2(self): @pytest.mark.asyncio async def test_pp_tinypose(self): - toolkit = RemoteToolkit.from_aistudio("pp-tinypose") + toolkit = RemoteToolkit.from_aistudio("pp-tinypose", file_manager=self.file_manager) agent = self.get_agent(toolkit) file_path = self.download_fixture_file("pp_tinypose_input_img.jpg") @@ -58,7 +58,7 @@ async def test_pp_tinypose(self): @pytest.mark.asyncio async def test_pp_vehicle(self): - toolkit = RemoteToolkit.from_aistudio("pp-vehicle") + toolkit = RemoteToolkit.from_aistudio("pp-vehicle", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path(self.download_fixture_file("vehicle.jpg")) agent = self.get_agent(toolkit) @@ -76,7 +76,7 @@ async def test_pp_vehicle(self): @pytest.mark.asyncio async def test_pp_structure(self): - toolkit = RemoteToolkit.from_aistudio("pp-structure-v2") + toolkit = RemoteToolkit.from_aistudio("pp-structure-v2", file_manager=self.file_manager) agent = self.get_agent(toolkit) file = await self.file_manager.create_file_from_path(self.download_fixture_file("ocr_table.png")) @@ -86,7 +86,7 @@ async def test_pp_structure(self): @pytest.mark.asyncio async def test_pp_ocr_v4(self): - toolkit = RemoteToolkit.from_aistudio("pp-ocrv4") + toolkit = RemoteToolkit.from_aistudio("pp-ocrv4", file_manager=self.file_manager) file = await self.file_manager.create_file_from_path( self.download_fixture_file("ocr_example_input.png") diff --git a/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py b/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py index 501d20cca..261ff4e05 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_pp_shituv2.py @@ -11,7 +11,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_pp_shituv2(self): - toolkit = RemoteToolkit.from_aistudio("pp-shituv2") + toolkit = RemoteToolkit.from_aistudio("pp-shituv2", file_manager=self.file_manager) tools = toolkit.get_tools() print(tools[0].function_call_schema()) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_text_moderation.py b/erniebot-agent/tests/integration_tests/apihub/test_text_moderation.py index 2d1e3ea39..15f850304 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_text_moderation.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_text_moderation.py @@ -10,7 +10,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_tool(self): - toolkit = RemoteToolkit.from_aistudio("text-moderation") + toolkit = RemoteToolkit.from_aistudio("text-moderation", file_manager=self.file_manager) agent = self.get_agent(toolkit) result = await agent.async_run("请判断：“我爱我的家乡” 这句话是否合规") diff --git a/erniebot-agent/tests/integration_tests/apihub/test_text_to_speech.py b/erniebot-agent/tests/integration_tests/apihub/test_text_to_speech.py index 1f1a1ec4b..0fe73013f 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_text_to_speech.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_text_to_speech.py @@ -10,7 +10,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_text_to_speech(self): - toolkit = RemoteToolkit.from_aistudio("texttospeech") + toolkit = RemoteToolkit.from_aistudio("texttospeech", file_manager=self.file_manager) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/integration_tests/apihub/test_translation.py b/erniebot-agent/tests/integration_tests/apihub/test_translation.py index c25e183f3..5cf71b37c 100644 --- a/erniebot-agent/tests/integration_tests/apihub/test_translation.py +++ b/erniebot-agent/tests/integration_tests/apihub/test_translation.py @@ -10,7 +10,7 @@ class TestRemoteTool(RemoteToolTesting): @pytest.mark.asyncio async def test_tool(self): - toolkit = RemoteToolkit.from_aistudio("translation") + toolkit = RemoteToolkit.from_aistudio("translation", file_manager=self.file_manager) agent = self.get_agent(toolkit) diff --git a/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py b/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py index e14819231..15ebcf957 100644 --- a/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py +++ b/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py @@ -1,7 +1,7 @@ import contextlib -from erniebot_agent.file_io import protocol -from erniebot_agent.file_io.remote_file import RemoteFile, RemoteFileClient +from erniebot_agent.file import protocol +from erniebot_agent.file.remote_file import RemoteFile, RemoteFileClient class FakeRemoteFileClient(RemoteFileClient): From d3723952ceac7d4a5ed206dc23b2b0196efa16ca Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Fri, 22 Dec 2023 19:30:50 +0800 Subject: [PATCH 26/63] Remove unused file --- erniebot-agent/test_concurrent.py | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 erniebot-agent/test_concurrent.py diff --git a/erniebot-agent/test_concurrent.py b/erniebot-agent/test_concurrent.py deleted file mode 100644 index e0f31764a..000000000 --- a/erniebot-agent/test_concurrent.py +++ /dev/null @@ -1,10 +0,0 @@ -import asyncio -from erniebot_agent.file import GlobalFileManagerHandler - -async def fun(): - await GlobalFileManagerHandler().get() - -async def main(): - await asyncio.gather(fun(), fun(), fun(), fun()) - -asyncio.run(main()) From 5e3273d8965fc477f0037ebe8fd4fbd4ea359ce0 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Sun, 24 Dec 2023 23:45:50 +0800 Subject: [PATCH 27/63] Fix and enhance --- .../agents/callback/callback_manager.py | 16 ++--- .../src/erniebot_agent/file/file_manager.py | 63 +++++++++++-------- .../src/erniebot_agent/file/file_registry.py | 5 +- .../file/global_file_manager_handler.py | 42 +++++++++---- .../src/erniebot_agent/file/protocol.py | 5 +- .../src/erniebot_agent/file/remote_file.py | 2 +- .../src/erniebot_agent/memory/messages.py | 2 +- .../src/erniebot_agent/tools/remote_tool.py | 5 +- .../erniebot_agent/tools/remote_toolkit.py | 5 +- .../src/erniebot_agent/utils/misc.py | 18 ++---- .../src/erniebot_agent/utils/mixins.py | 16 ++++- .../agents/callback/test_callback_manager.py | 9 ++- .../mocks/mock_remote_file_client_server.py | 7 ++- erniebot/README.md | 2 +- erniebot/docs/README.md | 2 +- erniebot/docs/authentication.md | 6 +- .../function_calling/function_calling_demo.py | 9 ++- erniebot/src/erniebot/__init__.py | 6 +- erniebot/src/erniebot/auth.py | 8 +-- erniebot/src/erniebot/cli.py | 4 +- erniebot/src/erniebot/config.py | 15 +++-- erniebot/src/erniebot/constants.py | 2 - erniebot/src/erniebot/errors.py | 7 +++ erniebot/src/erniebot/http_client.py | 20 +----- .../src/erniebot/resources/chat_completion.py | 4 +- .../resources/chat_completion_with_plugins.py | 4 +- erniebot/src/erniebot/resources/embedding.py | 4 +- .../src/erniebot/resources/fine_tuning.py | 14 ++--- erniebot/src/erniebot/resources/image.py | 14 ++--- erniebot/src/erniebot/resources/resource.py | 10 +-- erniebot/src/erniebot/response.py | 12 ++-- erniebot/src/erniebot/utils/misc.py | 6 +- erniebot/tests/README.md | 2 +- 33 files changed, 185 insertions(+), 161 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/agents/callback/callback_manager.py b/erniebot-agent/src/erniebot_agent/agents/callback/callback_manager.py index cd1247239..ff7ba16e5 100644 --- a/erniebot-agent/src/erniebot_agent/agents/callback/callback_manager.py +++ b/erniebot-agent/src/erniebot_agent/agents/callback/callback_manager.py @@ -32,30 +32,24 @@ class CallbackManager(object): def __init__(self, handlers: List[CallbackHandler]): super().__init__() - self._handlers = handlers + self._handlers: List[CallbackHandler] = [] + self.set_handlers(handlers) @property def handlers(self) -> List[CallbackHandler]: return self._handlers def add_handler(self, handler: CallbackHandler): - if handler in self._handlers: - raise ValueError(f"The callback handler {handler} is already registered.") self._handlers.append(handler) def remove_handler(self, handler): - try: - self._handlers.remove(handler) - except ValueError as e: - raise ValueError(f"The callback handler {handler} is not registered.") from e + self._handlers.remove(handler) def set_handlers(self, handlers: List[CallbackHandler]): - self._handlers = [] - for handler in handlers: - self.add_handler(handler) + self._handlers[:] = handlers def remove_all_handlers(self): - self._handlers = [] + self._handlers.clear() async def handle_event(self, event_type: EventType, *args: Any, **kwargs: Any) -> None: callback_name = "on_" + event_type.value diff --git a/erniebot-agent/src/erniebot_agent/file/file_manager.py b/erniebot-agent/src/erniebot_agent/file/file_manager.py index 26f08a998..a950845b8 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/file_manager.py @@ -17,8 +17,20 @@ import pathlib import tempfile import uuid +from collections import deque from types import TracebackType -from typing import Any, Dict, List, Literal, Optional, Type, Union, final, overload +from typing import ( + Any, + Deque, + Dict, + List, + Literal, + Optional, + Type, + Union, + final, + overload, +) import anyio from typing_extensions import Self, TypeAlias @@ -29,7 +41,7 @@ from erniebot_agent.file.local_file import LocalFile, create_local_file_from_path from erniebot_agent.file.remote_file import RemoteFile, RemoteFileClient from erniebot_agent.utils.exceptions import FileError -from erniebot_agent.utils.mixins import Closeable +from erniebot_agent.utils.mixins import Closeable, Noncopyable logger = logging.getLogger(__name__) @@ -37,7 +49,7 @@ @final -class FileManager(Closeable): +class FileManager(Closeable, Noncopyable): _temp_dir: Optional[tempfile.TemporaryDirectory] = None def __init__( @@ -45,7 +57,6 @@ def __init__( remote_file_client: Optional[RemoteFileClient] = None, save_dir: Optional[FilePath] = None, *, - default_file_type: Optional[Literal["local", "remote"]] = None, prune_on_close: bool = True, ) -> None: super().__init__() @@ -57,21 +68,13 @@ def __init__( # This can be done lazily, but we need to be careful about race conditions. self._temp_dir = self._create_temp_dir() self._save_dir = pathlib.Path(self._temp_dir.name) - self._default_file_type = default_file_type self._prune_on_close = prune_on_close self._file_registry = FileRegistry() - self._fully_managed_files: List[Union[LocalFile, RemoteFile]] = [] + self._fully_managed_files: Deque[Union[LocalFile, RemoteFile]] = deque() self._closed = False - @property - def remote_file_client(self) -> RemoteFileClient: - if self._remote_file_client is None: - raise AttributeError("No remote file client is set.") - else: - return self._remote_file_client - @property def closed(self): return self._closed @@ -137,7 +140,7 @@ async def create_file_from_path( elif file_type == "remote": file = await self.create_remote_file_from_path(file_path, file_purpose, file_metadata) else: - raise RuntimeError(f"Unsupported file type: {file_type}") + raise ValueError(f"Unsupported file type: {file_type}") return file async def create_local_file_from_path( @@ -238,7 +241,7 @@ async def create_file_from_bytes( file_metadata, ) else: - raise RuntimeError(f"Unsupported file type: {file_type}") + raise ValueError(f"Unsupported file type: {file_type}") finally: if should_remove_file: await async_file_path.unlink() @@ -248,13 +251,13 @@ async def create_file_from_bytes( async def retrieve_remote_file_by_id(self, file_id: str) -> RemoteFile: self.ensure_not_closed() - file = await self.remote_file_client.retrieve_file(file_id) + file = await self._get_remote_file_client().retrieve_file(file_id) self._file_registry.register_file(file) return file async def list_remote_files(self) -> List[RemoteFile]: self.ensure_not_closed() - files = await self.remote_file_client.list_files() + files = await self._get_remote_file_client().list_files() return files def look_up_file_by_id(self, file_id: str) -> File: @@ -272,18 +275,21 @@ def list_registered_files(self) -> List[File]: return self._file_registry.list_files() async def prune(self) -> None: - for file in self._fully_managed_files: + while True: + try: + file = self._fully_managed_files.pop() + except IndexError: + break if isinstance(file, RemoteFile): # FIXME: Currently this is not supported. # await file.delete() pass elif isinstance(file, LocalFile): - assert self._save_dir in file.path.parents + assert self._save_dir.resolve() in file.path.resolve().parents await anyio.Path(file.path).unlink() else: raise AssertionError("Unexpected file type") self._file_registry.unregister_file(file) - self._fully_managed_files.clear() async def close(self) -> None: if not self._closed: @@ -313,17 +319,20 @@ async def _create_remote_file_from_path( file_purpose: protocol.FilePurpose, file_metadata: Optional[Dict[str, Any]], ) -> RemoteFile: - file = await self.remote_file_client.upload_file(file_path, file_purpose, file_metadata or {}) + file = await self._get_remote_file_client().upload_file(file_path, file_purpose, file_metadata or {}) return file + def _get_remote_file_client(self) -> RemoteFileClient: + if self._remote_file_client is None: + raise AttributeError("No remote file client is set.") + else: + return self._remote_file_client + def _get_default_file_type(self) -> Literal["local", "remote"]: - if self._default_file_type is not None: - return self._default_file_type + if self._remote_file_client is not None: + return "remote" else: - if self._remote_file_client is not None: - return "remote" - else: - return "local" + return "local" def _get_unique_file_path( self, prefix: Optional[str] = None, suffix: Optional[str] = None diff --git a/erniebot-agent/src/erniebot_agent/file/file_registry.py b/erniebot-agent/src/erniebot_agent/file/file_registry.py index 08b7bf322..f3de22a9a 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_registry.py +++ b/erniebot-agent/src/erniebot_agent/file/file_registry.py @@ -23,14 +23,11 @@ def __init__(self) -> None: super().__init__() self._id_to_file: Dict[str, File] = {} - def register_file(self, file: File, *, allow_overwrite: bool = False, check_type: bool = True) -> None: + def register_file(self, file: File, *, allow_overwrite: bool = False) -> None: file_id = file.id if file_id in self._id_to_file: if not allow_overwrite: raise ValueError(f"File with ID {repr(file_id)} is already registered.") - else: - if check_type and type(file) is not type(self._id_to_file[file_id]): # noqa: E721 - raise RuntimeError("Cannot register a file with a different type.") self._id_to_file[file_id] = file def unregister_file(self, file: File) -> None: diff --git a/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py b/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py index 4d368179d..d4d809276 100644 --- a/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py +++ b/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py @@ -13,7 +13,7 @@ # limitations under the License. import asyncio -from typing import Any, Optional, final +from typing import Any, NoReturn, Optional, final import asyncio_atexit # type: ignore @@ -24,7 +24,7 @@ @final -class GlobalFileManagerHandler(metaclass=Singleton): +class GlobalFileManagerHandler(Singleton): _file_manager: Optional[FileManager] def __init__(self) -> None: @@ -36,30 +36,41 @@ async def get(self) -> FileManager: async with self._lock: if self._file_manager is None: self._file_manager = await self._create_default_file_manager( - access_token=None, save_dir=None + access_token=None, + save_dir=None, + enable_remote_file=False, ) return self._file_manager async def configure( self, + *, access_token: Optional[str] = None, save_dir: Optional[str] = None, + enable_remote_file: bool = False, **opts: Any, ) -> None: async with self._lock: if self._file_manager is not None: - raise RuntimeError( - "`GlobalFileManagerHandler.configure` can only be called once" - " and must be called before calling `GlobalFileManagerHandler.get`." - ) + self._raise_file_manager_already_set_error() self._file_manager = await self._create_default_file_manager( - access_token=access_token, save_dir=save_dir, **opts + access_token=access_token, + save_dir=save_dir, + enable_remote_file=enable_remote_file, + **opts, ) + async def set(self, file_manager: FileManager) -> None: + async with self._lock: + if self._file_manager is not None: + self._raise_file_manager_already_set_error() + self._file_manager = file_manager + async def _create_default_file_manager( self, access_token: Optional[str], save_dir: Optional[str], + enable_remote_file: bool, **opts: Any, ) -> FileManager: async def _close_file_manager(): @@ -69,10 +80,19 @@ async def _close_file_manager(): access_token = C.get_global_access_token() if save_dir is None: save_dir = C.get_global_save_dir() - if access_token is not None: + remote_file_client = None + if enable_remote_file: + if access_token is None: + raise RuntimeError("An access token must be provided to enable remote file management.") remote_file_client = AIStudioFileClient(access_token=access_token) - else: - remote_file_client = None file_manager = FileManager(remote_file_client, save_dir, **opts) asyncio_atexit.register(_close_file_manager) return file_manager + + def _raise_file_manager_already_set_error(self) -> NoReturn: + raise RuntimeError( + "The global file manager can only be set once." + " The setup can be done explicitly by calling" + " `GlobalFileManagerHandler.configure` or `GlobalFileManagerHandler.set`," + " or implicitly by calling `GlobalFileManagerHandler.get`." + ) diff --git a/erniebot-agent/src/erniebot_agent/file/protocol.py b/erniebot-agent/src/erniebot_agent/file/protocol.py index 6caad1424..149c6c5b5 100644 --- a/erniebot-agent/src/erniebot_agent/file/protocol.py +++ b/erniebot-agent/src/erniebot_agent/file/protocol.py @@ -65,4 +65,7 @@ def extract_remote_file_ids(str_: str) -> List[str]: def generate_fake_remote_file_ids() -> Generator[str, None, None]: counter = 0 while True: - yield _REMOTE_FILE_ID_PREFIX + f"{counter:015d}" + number = f"{counter:015d}" + if len(number) > 15: + break + yield _REMOTE_FILE_ID_PREFIX + number diff --git a/erniebot-agent/src/erniebot_agent/file/remote_file.py b/erniebot-agent/src/erniebot_agent/file/remote_file.py index 774a7a271..f9d851445 100644 --- a/erniebot-agent/src/erniebot_agent/file/remote_file.py +++ b/erniebot-agent/src/erniebot_agent/file/remote_file.py @@ -188,7 +188,7 @@ async def list_files(self) -> List[RemoteFile]: return files async def delete_file(self, file_id: str) -> None: - raise FileError(f"`{self.__class__.__name__}.{inspect.stack()[0][3]}` is not supported.") + raise TypeError(f"`{self.__class__.__name__}.{inspect.stack()[0][3]}` is not supported.") async def create_temporary_url(self, file_id: str, expire_after: float) -> str: url = self._get_url(self._RETRIEVE_ENDPOINT).format(file_id=file_id) diff --git a/erniebot-agent/src/erniebot_agent/memory/messages.py b/erniebot-agent/src/erniebot_agent/memory/messages.py index b173bb0e4..d9fb3cade 100644 --- a/erniebot-agent/src/erniebot_agent/memory/messages.py +++ b/erniebot-agent/src/erniebot_agent/memory/messages.py @@ -203,7 +203,7 @@ async def _create_file_reprs_with_urls(files: List[File]) -> List[str]: file_reprs = [] for file in files: if not isinstance(file, RemoteFile): - raise RuntimeError("Only `RemoteFile` objects can have URLs in their representations.") + raise TypeError("Only `RemoteFile` objects can have URLs in their representations.") url = await file.create_temporary_url() file_reprs.append(file.get_file_repr_with_url(url)) diff --git a/erniebot-agent/src/erniebot_agent/tools/remote_tool.py b/erniebot-agent/src/erniebot_agent/tools/remote_tool.py index 11421e103..29cce01ac 100644 --- a/erniebot-agent/src/erniebot_agent/tools/remote_tool.py +++ b/erniebot-agent/src/erniebot_agent/tools/remote_tool.py @@ -1,6 +1,7 @@ from __future__ import annotations import base64 +import dataclasses import json from copy import deepcopy from typing import Any, Dict, List, Optional, Type @@ -59,7 +60,9 @@ def __init__( self.tool_name_prefix = tool_name_prefix # If `tool_name_prefix`` is provided, we prepend `tool_name_prefix`` to the `name` field of all tools if tool_name_prefix is not None and not self.tool_view.name.startswith(f"{self.tool_name_prefix}/"): - self.tool_view.name = f"{self.tool_name_prefix}/{self.tool_view.name}" + self.tool_view = dataclasses.replace( + self.tool_view, name=f"{self.tool_name_prefix}/{self.tool_view.name}" + ) self.response_prompt: Optional[str] = None diff --git a/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py b/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py index db765085f..7455be481 100644 --- a/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py +++ b/erniebot-agent/src/erniebot_agent/tools/remote_toolkit.py @@ -1,6 +1,5 @@ from __future__ import annotations -import copy import json import logging import os @@ -61,7 +60,7 @@ def get_tools(self) -> List[RemoteTool]: TOOL_CLASS = tool_registor.get_tool_class(self.info.title) return [ TOOL_CLASS( - copy.deepcopy(path), + path, self.servers[0].url, self.headers, self.info.version, @@ -131,7 +130,7 @@ def get_tool(self, tool_name: str) -> RemoteTool: TOOL_CLASS = tool_registor.get_tool_class(self.info.title) return TOOL_CLASS( - copy.deepcopy(paths[0]), + paths[0], self.servers[0].url, self.headers, self.info.version, diff --git a/erniebot-agent/src/erniebot_agent/utils/misc.py b/erniebot-agent/src/erniebot_agent/utils/misc.py index 10092603c..4aa3f6574 100644 --- a/erniebot-agent/src/erniebot_agent/utils/misc.py +++ b/erniebot-agent/src/erniebot_agent/utils/misc.py @@ -12,19 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import threading -from typing import ClassVar - __all__ = ["Singleton"] -class Singleton(type): - _insts: ClassVar[dict] = {} - _lock = threading.Lock() +class Singleton(object): + _instance = None - def __call__(cls, *args, **kwargs): - if cls not in cls._insts: - with cls._lock: - if cls not in cls._insts: - cls._insts[cls] = super().__call__(*args, **kwargs) - return cls._insts[cls] + def __new__(cls, *args, **kwargs): + if cls._instance is not None: + cls._instance = super().__new__(*args, **kwargs) + return cls._instance diff --git a/erniebot-agent/src/erniebot_agent/utils/mixins.py b/erniebot-agent/src/erniebot_agent/utils/mixins.py index 21fe4f335..ad005056c 100644 --- a/erniebot-agent/src/erniebot_agent/utils/mixins.py +++ b/erniebot-agent/src/erniebot_agent/utils/mixins.py @@ -17,7 +17,7 @@ import base64 import os import tempfile -from typing import TYPE_CHECKING, Any, List, Optional, Protocol, cast +from typing import TYPE_CHECKING, Any, List, NoReturn, Optional, Protocol, cast, final from erniebot_agent.utils.common import get_file_type from erniebot_agent.utils.exceptions import ObjectClosedError @@ -242,9 +242,23 @@ class Closeable(Protocol): def closed(self) -> bool: ... + def __del__(self) -> None: + # TODO: Issue a `ResourceWarning` if the object is not closed. + return None + async def close(self) -> None: ... def ensure_not_closed(self) -> None: if self.closed: raise ObjectClosedError(f"{repr(self)} is closed.") + + +class Noncopyable(object): + @final + def __copy__(self) -> NoReturn: + raise TypeError("Cannot copy an instance of a non-copyable class.") + + @final + def __deepcopy__(self) -> NoReturn: + raise TypeError("Cannot deep-copy an instance of a non-copyable class.") diff --git a/erniebot-agent/tests/unit_tests/agents/callback/test_callback_manager.py b/erniebot-agent/tests/unit_tests/agents/callback/test_callback_manager.py index 48cffd004..fc1264e84 100644 --- a/erniebot-agent/tests/unit_tests/agents/callback/test_callback_manager.py +++ b/erniebot-agent/tests/unit_tests/agents/callback/test_callback_manager.py @@ -71,9 +71,6 @@ async def test_callback_manager_add_remove_handlers(): assert len(callback_manager.handlers) == 1 - with pytest.raises(ValueError): - callback_manager.add_handler(handler1) - callback_manager.remove_handler(handler1) assert len(callback_manager.handlers) == 0 @@ -83,5 +80,11 @@ async def test_callback_manager_add_remove_handlers(): callback_manager.add_handler(handler2) assert len(callback_manager.handlers) == 2 + callback_manager.add_handler(handler1) + assert len(callback_manager.handlers) == 3 + + callback_manager.remove_handler(handler1) + assert len(callback_manager.handlers) == 2 + callback_manager.remove_all_handlers() assert len(callback_manager.handlers) == 0 diff --git a/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py b/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py index 15ebcf957..4cbb702ba 100644 --- a/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py +++ b/erniebot-agent/tests/unit_tests/testing_utils/mocks/mock_remote_file_client_server.py @@ -38,7 +38,7 @@ async def delete_file(self, file_id) -> None: await self.server.delete_file(file_id) async def create_temporary_url(self, file_id, expire_after): - raise RuntimeError("Method not supported") + raise TypeError("Method not supported") def _create_file_obj_from_dict(self, dict_): return RemoteFile( @@ -69,7 +69,10 @@ def started(self): async def upload_file(self, file_path, file_purpose, file_metadata): if not self.started: raise ServerError("Server is not running.") - id_ = next(self._file_id_iter) + try: + id_ = next(self._file_id_iter) + except StopIteration as e: + raise ServerError("No more file IDs available") from e filename = file_path.name byte_size = file_path.stat().st_size created_at = protocol.get_timestamp() diff --git a/erniebot/README.md b/erniebot/README.md index dcd641df0..0356e121b 100644 --- a/erniebot/README.md +++ b/erniebot/README.md @@ -208,7 +208,7 @@ response = erniebot.ChatCompletion.create( "role": "user", "content": "深圳市今天气温多少摄氏度？", }, ], - functions = [ + functions=[ { "name": "get_current_temperature", "description": "获取指定城市的气温", diff --git a/erniebot/docs/README.md b/erniebot/docs/README.md index 8d860bea3..27e829694 100644 --- a/erniebot/docs/README.md +++ b/erniebot/docs/README.md @@ -171,7 +171,7 @@ response = erniebot.ChatCompletion.create( "role": "user", "content": "深圳市今天气温多少摄氏度？", }, ], - functions = [ + functions=[ { "name": "get_current_temperature", "description": "获取指定城市的气温", diff --git a/erniebot/docs/authentication.md b/erniebot/docs/authentication.md index 5a029a3e7..3d471433a 100644 --- a/erniebot/docs/authentication.md +++ b/erniebot/docs/authentication.md @@ -14,7 +14,7 @@ ERNIE Bot SDK支持多个后端平台来调用文心大模型（如下表格） ### 申请用户凭证 -在[AI Studio星河社区](https://aistudio.baidu.com/index)注册并登录账号，可以在个人中心的[访问令牌页面](https://aistudio.baidu.com/usercenter/token)获取用户凭证access token。 +在[AI Studio星河社区](https://aistudio.baidu.com/index)注册并登录账号，可以在个人中心的[访问令牌页面](https://aistudio.baidu.com/usercenter/token)获取access token。
    @@ -145,9 +145,9 @@ AI Studio后端可以使用access token进行鉴权，支持如下三种方法 ### 设置鉴权参数 -为智能创作平台后端设置鉴权参数的方法，和千帆后端完全一致，都支持access toke或者API key+secret key。举例如下。 +为智能创作平台后端设置鉴权参数的方法，和千帆后端完全一致，都支持access token或者API key+secret key。举例如下。 -请注意设置后端参数为`"yinian"`，并且使用智能创作平台申请的access_token、API key、secret key。 +请注意设置后端参数为`"yinian"`，并且使用智能创作平台申请的access token、API key、secret key。 1. 使用access token的例子： diff --git a/erniebot/examples/function_calling/function_calling_demo.py b/erniebot/examples/function_calling/function_calling_demo.py index 9f2a54a95..2db2f435e 100644 --- a/erniebot/examples/function_calling/function_calling_demo.py +++ b/erniebot/examples/function_calling/function_calling_demo.py @@ -16,7 +16,6 @@ import argparse import collections -import copy import functools import inspect import json @@ -530,7 +529,7 @@ def recall_message(state): raise gr.Error("请至少进行一轮对话") context = context[:-2] history = extract_history(context) - state["context"] = context + state["context"][:] = context return state, history, context[-MAX_CONTEXT_LINES_TO_SHOW:] @@ -572,7 +571,7 @@ def generate_response( top_p, temperature, ): - context = copy.copy(state["context"]) + context = state["context"].copy() context.append(message) name2function = state["name2function"] functions = [name2function[name].desc for name in candidates] @@ -610,7 +609,7 @@ def generate_response( context[-1]["function_call"] = function_call assert history is None history = extract_history(context) - state["context"] = context + state["context"][:] = context yield ( state, history, @@ -643,7 +642,7 @@ def generate_response( ) assert history[-1][1] == context[-1]["content"] else: - state["context"] = context + state["context"][:] = context yield ( state, history, diff --git a/erniebot/src/erniebot/__init__.py b/erniebot/src/erniebot/__init__.py index 724ce40ed..c9ad7da36 100644 --- a/erniebot/src/erniebot/__init__.py +++ b/erniebot/src/erniebot/__init__.py @@ -13,8 +13,8 @@ # limitations under the License. from . import errors -from .config import GlobalConfig -from .config import init_global_config as _init_global_config +from .errors import ConfigItemNotFoundError as _ConfigItemNotFoundError +from .config import GlobalConfig, init_global_config as _init_global_config from .intro import Model from .resources import ( ChatCompletion, @@ -65,5 +65,5 @@ def __getattr__(name): # erniebot/config.py. try: return GlobalConfig().get_value(name) - except KeyError: + except _ConfigItemNotFoundError: raise AttributeError(f"module '{__name__}' has no attribute '{name}'") from None diff --git a/erniebot/src/erniebot/auth.py b/erniebot/src/erniebot/auth.py index 64f3f9a0a..058d72b35 100644 --- a/erniebot/src/erniebot/auth.py +++ b/erniebot/src/erniebot/auth.py @@ -25,7 +25,7 @@ from . import errors from .api_types import APIType from .utils import logging -from .utils.misc import Singleton +from .utils.misc import SingletonMeta __all__ = ["build_auth_token_manager"] @@ -37,7 +37,7 @@ def build_auth_token_manager(manager_type: str, api_type: APIType, **kwargs: Any raise ValueError(f"Unsupported manager type: {manager_type}") -class _GlobalAuthTokenCache(metaclass=Singleton): +class _GlobalAuthTokenCache(metaclass=SingletonMeta): _MIN_UPDATE_INTERVAL_SECS: ClassVar[float] = 3600 @dataclass @@ -116,7 +116,7 @@ def get_auth_token(self) -> str: def update_auth_token(self) -> str: new_token = self._update_cache(init=False) self._token = new_token - logging.info("Security token is updated.") + logging.info("Security token has been updated.") return self._token def _request_auth_token(self, init: bool) -> str: @@ -147,7 +147,7 @@ def _update_cache(self, init: bool) -> str: functools.partial(self._request_auth_token, init=init), ) if upserted: - logging.debug("Cache is updated.") + logging.debug("Cache updated") return token diff --git a/erniebot/src/erniebot/cli.py b/erniebot/src/erniebot/cli.py index 84c752b3f..62138b980 100644 --- a/erniebot/src/erniebot/cli.py +++ b/erniebot/src/erniebot/cli.py @@ -118,10 +118,10 @@ def get_resource_class(cls): def register_api_to_parser(cls, parser, api_name): def _find_method(method_name): if not hasattr(cls, method_name): - raise RuntimeError(f"{cls.__name__}.{method_name} is not found.") + raise RuntimeError(f"`{cls.__name__}.{method_name}` does not exist.") method = getattr(cls, method_name) if not callable(method): - raise RuntimeError(f"{cls.__name__}.{method_name} is not callable.") + raise RuntimeError(f"`{cls.__name__}.{method_name}` is not callable.") return method cls.add_resource_arguments(parser) diff --git a/erniebot/src/erniebot/config.py b/erniebot/src/erniebot/config.py index 227678630..a527f69a2 100644 --- a/erniebot/src/erniebot/config.py +++ b/erniebot/src/erniebot/config.py @@ -21,7 +21,8 @@ from typing import Any, Dict, Optional from .types import ConfigDictType -from .utils.misc import Singleton +from .utils.misc import SingletonMeta +from .errors import ConfigItemNotFoundError __all__ = ["GlobalConfig", "init_global_config"] @@ -71,15 +72,21 @@ def add_item(self, cfg: "_ConfigItem") -> None: self._cfg_dict[cfg.key] = cfg def get_value(self, key: str) -> Any: - cfg = self._cfg_dict[key] + try: + cfg = self._cfg_dict[key] + except KeyError as e: + raise ConfigItemNotFoundError from e return cfg.value def set_value(self, key: str, value: Any) -> None: - cfg = self._cfg_dict[key] + try: + cfg = self._cfg_dict[key] + except KeyError as e: + raise ConfigItemNotFoundError from e cfg.value = value -class GlobalConfig(_Config, metaclass=Singleton): +class GlobalConfig(_Config, metaclass=SingletonMeta): def create_dict(self, **overrides: Any) -> ConfigDictType: dict_: ConfigDictType = {} for key, cfg in self._cfg_dict.items(): diff --git a/erniebot/src/erniebot/constants.py b/erniebot/src/erniebot/constants.py index 9cc34a223..eb3c20ed8 100644 --- a/erniebot/src/erniebot/constants.py +++ b/erniebot/src/erniebot/constants.py @@ -17,8 +17,6 @@ STREAM_RESPONSE_PREFIX: bytes = b"data: " DEFAULT_REQUEST_TIMEOUT_SECS: float = 600 -MAX_CONNECTION_RETRIES: int = 2 -MAX_SESSION_LIFETIME_SECS: float = 180 POLLING_INTERVAL_SECS: float = 5 POLLING_TIMEOUT_SECS: float = 20 diff --git a/erniebot/src/erniebot/errors.py b/erniebot/src/erniebot/errors.py index 4bd5ce52e..c664c8c96 100644 --- a/erniebot/src/erniebot/errors.py +++ b/erniebot/src/erniebot/errors.py @@ -39,6 +39,13 @@ class EBError(Exception): class ArgumentNotFoundError(EBError): """An argument was not found.""" + def __init__(self, argument: str) -> None: + super().__init__(f"Argument `{argument}` not found") + + +class ConfigItemNotFoundError(EBError): + """A configuration item was not found.""" + class InvalidArgumentError(EBError): """An argument is invalid.""" diff --git a/erniebot/src/erniebot/http_client.py b/erniebot/src/erniebot/http_client.py index 25262aa2a..8cfa5e920 100644 --- a/erniebot/src/erniebot/http_client.py +++ b/erniebot/src/erniebot/http_client.py @@ -76,8 +76,6 @@ class EBClient(object): """Provides low-level APIs to send HTTP requests and handle responses.""" - MAX_CONNECTION_RETRIES: ClassVar[int] = constants.MAX_CONNECTION_RETRIES - MAX_SESSION_LIFETIME_SECS: ClassVar[float] = constants.MAX_SESSION_LIFETIME_SECS DEFAULT_REQUEST_TIMEOUT_SECS: ClassVar[float] = constants.DEFAULT_REQUEST_TIMEOUT_SECS _session: Optional[requests.Session] @@ -308,9 +306,6 @@ async def asend_request_raw( "data": data, "timeout": timeout, } - proxy = self._proxy - if proxy is not None: - request_kwargs["proxy"] = proxy try: result = await session.request(method=method, url=url, **request_kwargs) @@ -478,14 +473,9 @@ def _make_requests_session_context_manager(self) -> Generator[requests.Session, session = requests.Session() should_close_session = True try: - proxies = self._get_proxies(self._proxy) - if proxies: - logging.debug("Use proxies: %r", proxies) + if self._proxy is not None: + proxies = {"http": self._proxy, "https": self._proxy} session.proxies = proxies - session.mount( - "https://", - requests.adapters.HTTPAdapter(max_retries=self.MAX_CONNECTION_RETRIES), - ) yield session finally: if should_close_session: @@ -507,9 +497,3 @@ async def _make_aiohttp_session_context_manager( finally: if should_close_session: await session.close() - - def _get_proxies(self, proxy: Optional[str]) -> Optional[Dict[str, str]]: - if proxy is None: - return None - else: - return {"http": proxy, "https": proxy} diff --git a/erniebot/src/erniebot/resources/chat_completion.py b/erniebot/src/erniebot/resources/chat_completion.py index 21c305383..83c6ad52c 100644 --- a/erniebot/src/erniebot/resources/chat_completion.py +++ b/erniebot/src/erniebot/resources/chat_completion.py @@ -447,7 +447,7 @@ def _set_val_if_key_exists(src: dict, dst: dict, key: str) -> None: # model if "model" not in kwargs: - raise errors.ArgumentNotFoundError("`model` is not found.") + raise errors.ArgumentNotFoundError("model") model = kwargs["model"] # For backward compatibility model = _update_model_name( @@ -462,7 +462,7 @@ def _set_val_if_key_exists(src: dict, dst: dict, key: str) -> None: # messages if "messages" not in kwargs: - raise errors.ArgumentNotFoundError("`messages` is not found.") + raise errors.ArgumentNotFoundError("messages") messages = kwargs["messages"] # path diff --git a/erniebot/src/erniebot/resources/chat_completion_with_plugins.py b/erniebot/src/erniebot/resources/chat_completion_with_plugins.py index 2c07557f1..ebdf80483 100644 --- a/erniebot/src/erniebot/resources/chat_completion_with_plugins.py +++ b/erniebot/src/erniebot/resources/chat_completion_with_plugins.py @@ -239,12 +239,12 @@ def _set_val_if_key_exists(src: dict, dst: dict, key: str) -> None: # messages if "messages" not in kwargs: - raise errors.ArgumentNotFoundError("`messages` is not found.") + raise errors.ArgumentNotFoundError("messages") messages = kwargs["messages"] # plugins if "plugins" not in kwargs: - raise errors.ArgumentNotFoundError("`plugins` is not found.") + raise errors.ArgumentNotFoundError("plugins") plugins = kwargs["plugins"] # path diff --git a/erniebot/src/erniebot/resources/embedding.py b/erniebot/src/erniebot/resources/embedding.py index 8d05c50b2..f9465e027 100644 --- a/erniebot/src/erniebot/resources/embedding.py +++ b/erniebot/src/erniebot/resources/embedding.py @@ -145,12 +145,12 @@ def _set_val_if_key_exists(src: dict, dst: dict, key: str) -> None: # model if "model" not in kwargs: - raise errors.ArgumentNotFoundError("`model` is not found.") + raise errors.ArgumentNotFoundError("model") model = kwargs["model"] # input if "input" not in kwargs: - raise errors.ArgumentNotFoundError("`input` is not found.") + raise errors.ArgumentNotFoundError("input") input = kwargs["input"] # path diff --git a/erniebot/src/erniebot/resources/fine_tuning.py b/erniebot/src/erniebot/resources/fine_tuning.py index 84c290c98..b634544c3 100644 --- a/erniebot/src/erniebot/resources/fine_tuning.py +++ b/erniebot/src/erniebot/resources/fine_tuning.py @@ -88,12 +88,12 @@ def _prepare_create(self, kwargs: Dict[str, Any]) -> Request: # name if "name" not in kwargs: - raise errors.ArgumentNotFoundError("`name` is not found.") + raise errors.ArgumentNotFoundError("name") name = kwargs["name"] # description if "description" not in kwargs: - raise errors.ArgumentNotFoundError("`description` is not found.") + raise errors.ArgumentNotFoundError("description") description = kwargs["description"] # path @@ -287,7 +287,7 @@ async def acancel( def _prepare_create(self, kwargs: Dict[str, Any]) -> Request: def _get_required_arg(key: str) -> Any: if key not in kwargs: - raise errors.ArgumentNotFoundError(f"`{key}` is not found.") + raise errors.ArgumentNotFoundError(key) return kwargs[key] valid_keys = { @@ -369,12 +369,12 @@ def _prepare_query(self, kwargs: Dict[str, Any]) -> Request: # task_id if "task_id" not in kwargs: - raise errors.ArgumentNotFoundError("`task_id` is not found.") + raise errors.ArgumentNotFoundError("task_id") task_id = kwargs["task_id"] # job_id if "job_id" not in kwargs: - raise errors.ArgumentNotFoundError("`job_id` is not found.") + raise errors.ArgumentNotFoundError("job_id") job_id = kwargs["job_id"] # path @@ -413,12 +413,12 @@ def _prepare_cancel(self, kwargs: Dict[str, Any]) -> Request: # task_id if "task_id" not in kwargs: - raise errors.ArgumentNotFoundError("`task_id` is not found.") + raise errors.ArgumentNotFoundError("task_id") task_id = kwargs["task_id"] # job_id if "job_id" not in kwargs: - raise errors.ArgumentNotFoundError("`job_id` is not found.") + raise errors.ArgumentNotFoundError("job_id") job_id = kwargs["job_id"] # path diff --git a/erniebot/src/erniebot/resources/image.py b/erniebot/src/erniebot/resources/image.py index 28a2ea209..c7cea3b9f 100644 --- a/erniebot/src/erniebot/resources/image.py +++ b/erniebot/src/erniebot/resources/image.py @@ -175,17 +175,17 @@ def _set_val_if_key_exists(src: dict, dst: dict, key: str) -> None: # text if "text" not in kwargs: - raise errors.ArgumentNotFoundError("`text` is not found.") + raise errors.ArgumentNotFoundError("text") text = kwargs["text"] # resolution if "resolution" not in kwargs: - raise errors.ArgumentNotFoundError("`resolution` is not found.") + raise errors.ArgumentNotFoundError("resolution") resolution = kwargs["resolution"] # style if "style" not in kwargs: - raise errors.ArgumentNotFoundError("`style` is not found.") + raise errors.ArgumentNotFoundError("style") style = kwargs["style"] # path @@ -369,22 +369,22 @@ def _set_val_if_key_exists(src: dict, dst: dict, key: str) -> None: # model if "model" not in kwargs: - raise errors.ArgumentNotFoundError("`model` is not found.") + raise errors.ArgumentNotFoundError("model") model = kwargs["model"] # prompt if "prompt" not in kwargs: - raise errors.ArgumentNotFoundError("`prompt` is not found.") + raise errors.ArgumentNotFoundError("prompt") prompt = kwargs["prompt"] # width if "width" not in kwargs: - raise errors.ArgumentNotFoundError("`width` is not found.") + raise errors.ArgumentNotFoundError("width") width = kwargs["width"] # height if "height" not in kwargs: - raise errors.ArgumentNotFoundError("`height` is not found.") + raise errors.ArgumentNotFoundError("height") height = kwargs["height"] # path diff --git a/erniebot/src/erniebot/resources/resource.py b/erniebot/src/erniebot/resources/resource.py index 1a2422109..54bf794dc 100644 --- a/erniebot/src/erniebot/resources/resource.py +++ b/erniebot/src/erniebot/resources/resource.py @@ -369,13 +369,10 @@ def _request( if stream: if not isinstance(resp, Iterator): raise RuntimeError("Expected an iterator of response objects") - else: - return resp else: if not isinstance(resp, EBResponse): raise RuntimeError("Expected a response object") - else: - return resp + return resp @overload async def _arequest( @@ -439,13 +436,10 @@ async def _arequest( if stream: if not isinstance(resp, AsyncIterator): raise RuntimeError("Expected an iterator of response objects") - else: - return resp else: if not isinstance(resp, EBResponse): raise RuntimeError("Expected a response object") - else: - return resp + return resp def _create_config_dict(self, overrides: Any) -> ConfigDictType: cfg_dict = GlobalConfig().create_dict(**overrides) diff --git a/erniebot/src/erniebot/response.py b/erniebot/src/erniebot/response.py index 9e5d24aa5..6cb5eeab0 100644 --- a/erniebot/src/erniebot/response.py +++ b/erniebot/src/erniebot/response.py @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -import copy import inspect import json from collections.abc import Mapping @@ -137,7 +136,7 @@ def __setattr__(self, key: str, value: Any) -> None: raise AttributeError def __reduce__(self) -> tuple: - state = copy.copy(self._dict) + state = self._dict.copy() rcode = state.pop("rcode") rbody = state.pop("rbody") rheaders = state.pop("rheaders") @@ -149,11 +148,8 @@ def __setstate__(self, state: dict) -> None: def get_result(self) -> Any: return self.rbody - def to_dict(self, deep_copy: bool = False) -> Dict[str, Any]: - if deep_copy: - return copy.deepcopy(self._dict) - else: - return copy.copy(self._dict) + def to_dict(self) -> Dict[str, Any]: + return self._dict.copy() def to_json(self) -> str: return json.dumps(self._dict) @@ -162,6 +158,6 @@ def _update_from_dict(self, dict_: Dict[str, Any]) -> None: member_names = set(pair[0] for pair in inspect.getmembers(self)) for k, v in dict_.items(): if k in self._RESERVED_KEYS or k in member_names: - raise KeyError(f"{repr(k)} is a reserved key.") + raise ValueError(f"{repr(k)} is a reserved key.") else: self._dict[k] = v diff --git a/erniebot/src/erniebot/utils/misc.py b/erniebot/src/erniebot/utils/misc.py index a8f5a5f2a..f575c3a04 100644 --- a/erniebot/src/erniebot/utils/misc.py +++ b/erniebot/src/erniebot/utils/misc.py @@ -16,7 +16,7 @@ from collections.abc import AsyncIterator, Iterator from typing import ClassVar -__all__ = ["Constant", "Singleton", "NOT_GIVEN", "NotGiven", "filter_args", "transform"] +__all__ = ["Constant", "SingletonMeta", "NOT_GIVEN", "NotGiven", "filter_args", "transform"] class Constant(object): @@ -31,7 +31,7 @@ def __set__(self, obj, val): raise AttributeError("The value of a constant cannot be modified.") -class Singleton(type): +class SingletonMeta(type): _insts: ClassVar[dict] = {} _lock: ClassVar[threading.Lock] = threading.Lock() @@ -43,7 +43,7 @@ def __call__(cls, *args, **kwargs): return cls._insts[cls] -class _NotGivenSentinel(metaclass=Singleton): +class _NotGivenSentinel(metaclass=SingletonMeta): def __bool__(self): return False diff --git a/erniebot/tests/README.md b/erniebot/tests/README.md index 0ff835665..c8ee15cc4 100644 --- a/erniebot/tests/README.md +++ b/erniebot/tests/README.md @@ -1 +1 @@ -ERNIE Bot SDK Tests +ERNIE Bot SDK Smoke Tests From c0858c978898c4fa02ae354e528c5ad770179cbf Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Sun, 24 Dec 2023 23:50:41 +0800 Subject: [PATCH 28/63] Fix style --- erniebot/src/erniebot/__init__.py | 3 ++- erniebot/src/erniebot/config.py | 2 +- erniebot/src/erniebot/http_client.py | 1 - 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/erniebot/src/erniebot/__init__.py b/erniebot/src/erniebot/__init__.py index c9ad7da36..461603d24 100644 --- a/erniebot/src/erniebot/__init__.py +++ b/erniebot/src/erniebot/__init__.py @@ -13,8 +13,9 @@ # limitations under the License. from . import errors +from .config import GlobalConfig +from .config import init_global_config as _init_global_config from .errors import ConfigItemNotFoundError as _ConfigItemNotFoundError -from .config import GlobalConfig, init_global_config as _init_global_config from .intro import Model from .resources import ( ChatCompletion, diff --git a/erniebot/src/erniebot/config.py b/erniebot/src/erniebot/config.py index a527f69a2..08cafcdde 100644 --- a/erniebot/src/erniebot/config.py +++ b/erniebot/src/erniebot/config.py @@ -20,9 +20,9 @@ import types from typing import Any, Dict, Optional +from .errors import ConfigItemNotFoundError from .types import ConfigDictType from .utils.misc import SingletonMeta -from .errors import ConfigItemNotFoundError __all__ = ["GlobalConfig", "init_global_config"] diff --git a/erniebot/src/erniebot/http_client.py b/erniebot/src/erniebot/http_client.py index 8cfa5e920..7a0b5da61 100644 --- a/erniebot/src/erniebot/http_client.py +++ b/erniebot/src/erniebot/http_client.py @@ -50,7 +50,6 @@ AsyncIterator, Callable, ClassVar, - Dict, Generator, Iterator, Mapping, From 1e6c17ff3bbd13d4d2f07dbfe5a837fdf3258b6f Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Mon, 25 Dec 2023 00:59:39 +0800 Subject: [PATCH 29/63] Fix CI --- .github/workflows/agent_ci.yaml | 10 +++++----- .github/workflows/client_ci.yaml | 8 ++++---- erniebot-agent/Makefile | 17 +++++++++++------ erniebot/Makefile | 13 +++++++++---- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/.github/workflows/agent_ci.yaml b/.github/workflows/agent_ci.yaml index f9cf893af..dd80d74bd 100644 --- a/.github/workflows/agent_ci.yaml +++ b/.github/workflows/agent_ci.yaml @@ -34,14 +34,14 @@ jobs: python -m pip install -r erniebot-agent/dev-requirements.txt - name: Show make version run: make --version - - name: Format Python code - run: make format + - name: Perform format checks on Python code + run: make format-check working-directory: erniebot-agent - name: Lint Python code run: make lint working-directory: erniebot-agent - - name: Type-check Python code - run: make type_check + - name: Perform type checks on Python code + run: make type-check working-directory: erniebot-agent UnitTest: name: Unit Test @@ -69,7 +69,7 @@ jobs: - name: Show make version run: make --version - name: Run unit tests - run: make test_coverage + run: make coverage working-directory: erniebot-agent - name: Upload coverage reports to Codecov uses: codecov/codecov-action@v3 diff --git a/.github/workflows/client_ci.yaml b/.github/workflows/client_ci.yaml index 3e44e677a..67ec66715 100644 --- a/.github/workflows/client_ci.yaml +++ b/.github/workflows/client_ci.yaml @@ -33,9 +33,9 @@ jobs: python -m pip install -r dev-requirements.txt - name: Show make version run: make --version - - name: Format Python code - run: make format + - name: Perform format checks on Python code + run: make format-check - name: Lint Python code run: make lint - - name: Type-check Python code - run: make type_check + - name: Perform type checks on Python code + run: make type-check diff --git a/erniebot-agent/Makefile b/erniebot-agent/Makefile index 1f527f474..36b49869e 100644 --- a/erniebot-agent/Makefile +++ b/erniebot-agent/Makefile @@ -2,25 +2,30 @@ files_to_format_and_lint = src examples tests .PHONY: dev -dev: format lint type_check +dev: format lint type-check .PHONY: format format: python -m black $(files_to_format_and_lint) - python -m isort --filter-files $(files_to_format_and_lint) + python -m isort $(files_to_format_and_lint) + +.PHONY: format-check +format-check: + python -m black --check --diff $(files_to_format_and_lint) + python -m isort --check-only --diff $(files_to_format_and_lint) .PHONY: lint lint: python -m flake8 $(files_to_format_and_lint) -.PHONY: type_check -type_check: +.PHONY: type-check +type-check: python -m mypy src .PHONY: test test: python -m pytest tests/unit_tests -.PHONY: test_coverage -test_coverage: +.PHONY: coverage +coverage: python -m pytest tests/unit_tests --cov erniebot_agent --cov-report xml:coverage.xml diff --git a/erniebot/Makefile b/erniebot/Makefile index cd2c1f7ea..9f0f8b221 100644 --- a/erniebot/Makefile +++ b/erniebot/Makefile @@ -2,17 +2,22 @@ files_to_format_and_lint = src examples tests .PHONY: dev -dev: format lint type_check +dev: format lint type-check .PHONY: format format: python -m black $(files_to_format_and_lint) - python -m isort --filter-files $(files_to_format_and_lint) + python -m isort $(files_to_format_and_lint) + +.PHONY: format-check +format-check: + python -m black --check --diff $(files_to_format_and_lint) + python -m isort --check-only --diff $(files_to_format_and_lint) .PHONY: lint lint: python -m flake8 $(files_to_format_and_lint) -.PHONY: type_check -type_check: +.PHONY: type-check +type-check: python -m mypy src From f84514da8bd973241b1685242eebd5ed475478c4 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Mon, 25 Dec 2023 02:27:57 +0800 Subject: [PATCH 30/63] Update CI config --- .github/workflows/agent_ci.yaml | 14 ++++++-------- .github/workflows/client_ci.yaml | 8 ++++---- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/.github/workflows/agent_ci.yaml b/.github/workflows/agent_ci.yaml index dd80d74bd..d4e9cf656 100644 --- a/.github/workflows/agent_ci.yaml +++ b/.github/workflows/agent_ci.yaml @@ -22,16 +22,17 @@ jobs: with: python-version: 3.8 cache: pip # Caching pip dependencies - cache-dependency-path: | + cache-dependency-path: &dep_file_paths | erniebot/setup.cfg erniebot-agent/setup.cfg - erniebot-agent/dev-requirements.txt + erniebot-agent/*-requirements.txt + erniebot-agent/tests/requirements.txt - name: Install erniebot-agent and dependencies run: | python -m pip install --upgrade pip + python -m pip install -r erniebot-agent/dev-requirements.txt python -m pip install ./erniebot python -m pip install ./erniebot-agent[all] - python -m pip install -r erniebot-agent/dev-requirements.txt - name: Show make version run: make --version - name: Perform format checks on Python code @@ -56,16 +57,13 @@ jobs: with: python-version: 3.8 cache: pip # Caching pip dependencies - cache-dependency-path: | - erniebot/setup.cfg - erniebot-agent/setup.cfg - erniebot-agent/tests/requirements.txt + cache-dependency-path: *dep_file_paths - name: Install erniebot-agent and dependencies run: | python -m pip install --upgrade pip + python -m pip install -r erniebot-agent/tests/requirements.txt python -m pip install ./erniebot python -m pip install ./erniebot-agent[all] - python -m pip install -r erniebot-agent/tests/requirements.txt - name: Show make version run: make --version - name: Run unit tests diff --git a/.github/workflows/client_ci.yaml b/.github/workflows/client_ci.yaml index 67ec66715..ec449e5b1 100644 --- a/.github/workflows/client_ci.yaml +++ b/.github/workflows/client_ci.yaml @@ -23,14 +23,14 @@ jobs: with: python-version: 3.8 cache: pip # Caching pip dependencies - cache-dependency-path: | + cache-dependency-path: &dep_file_paths | setup.cfg - dev-requirements.txt - - name: Install dependencies + *-requirements.txt + - name: Install erniebot and dependencies run: | + python -m pip install -r dev-requirements.txt python -m pip install --upgrade pip python -m pip install . - python -m pip install -r dev-requirements.txt - name: Show make version run: make --version - name: Perform format checks on Python code From 275c572302808d589c8590eb5dc85f88dae54ae6 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Mon, 25 Dec 2023 02:29:03 +0800 Subject: [PATCH 31/63] Fix style --- erniebot-agent/src/erniebot_agent/tools/remote_tool.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erniebot-agent/src/erniebot_agent/tools/remote_tool.py b/erniebot-agent/src/erniebot_agent/tools/remote_tool.py index 4414882d9..6ef86507b 100644 --- a/erniebot-agent/src/erniebot_agent/tools/remote_tool.py +++ b/erniebot-agent/src/erniebot_agent/tools/remote_tool.py @@ -315,7 +315,7 @@ def __adhoc_post_process__(self, tool_response: dict) -> dict: result.pop(key) # Remove log_id if in tool_response if "log_id" in tool_response: - tool_response.pop("log_id") + tool_response.pop("log_id") return tool_response async def _get_file_manager(self) -> FileManager: From 357a28c240ace535468171bb56d97ca2f454dc58 Mon Sep 17 00:00:00 2001 From: Bobholamovic Date: Mon, 25 Dec 2023 02:31:57 +0800 Subject: [PATCH 32/63] Remove anchors --- .github/workflows/agent_ci.yaml | 8 ++++++-- .github/workflows/client_ci.yaml | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/agent_ci.yaml b/.github/workflows/agent_ci.yaml index d4e9cf656..eb331fda5 100644 --- a/.github/workflows/agent_ci.yaml +++ b/.github/workflows/agent_ci.yaml @@ -22,7 +22,7 @@ jobs: with: python-version: 3.8 cache: pip # Caching pip dependencies - cache-dependency-path: &dep_file_paths | + cache-dependency-path: | erniebot/setup.cfg erniebot-agent/setup.cfg erniebot-agent/*-requirements.txt @@ -57,7 +57,11 @@ jobs: with: python-version: 3.8 cache: pip # Caching pip dependencies - cache-dependency-path: *dep_file_paths + cache-dependency-path: | + erniebot/setup.cfg + erniebot-agent/setup.cfg + erniebot-agent/*-requirements.txt + erniebot-agent/tests/requirements.txt - name: Install erniebot-agent and dependencies run: | python -m pip install --upgrade pip diff --git a/.github/workflows/client_ci.yaml b/.github/workflows/client_ci.yaml index ec449e5b1..9b76bfc93 100644 --- a/.github/workflows/client_ci.yaml +++ b/.github/workflows/client_ci.yaml @@ -23,7 +23,7 @@ jobs: with: python-version: 3.8 cache: pip # Caching pip dependencies - cache-dependency-path: &dep_file_paths | + cache-dependency-path: | setup.cfg *-requirements.txt - name: Install erniebot and dependencies From b0ec207285eeb74ddf73d50fce5537e5da5b6891 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 09:45:14 +0800 Subject: [PATCH 33/63] update docstring v1 --- .../src/erniebot_agent/file/__init__.py | 4 ++- .../src/erniebot_agent/file/base.py | 1 + .../src/erniebot_agent/file/file_manager.py | 6 ++-- .../src/erniebot_agent/file/local_file.py | 28 +++++++++++++++++++ .../src/erniebot_agent/file/protocol.py | 7 +++++ 5 files changed, 41 insertions(+), 5 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/file/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py index de9325f5d..53f7d1d0d 100644 --- a/erniebot-agent/src/erniebot_agent/file/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file/__init__.py @@ -18,7 +18,9 @@ A few notes about the current state of this submodule: - If you do not set environment variable `EB_ACCESS_TOKEN`, it will be under default setting. - Method `configure_global_file_manager` can only be called once at the beginning. -- When you want to get file_manger, you can use method `get_global_file_manager`. +- When you want to get a file manger, you can use method `get_global_file_manager`. +- If you want to get the content of `File` object, you can use `read_contents` + and use `write_contents_to` create the file to location you want. """ from erniebot_agent.file.global_file_manager_handler import GlobalFileManagerHandler diff --git a/erniebot-agent/src/erniebot_agent/file/base.py b/erniebot-agent/src/erniebot_agent/file/base.py index d9b509258..951a11fc9 100644 --- a/erniebot-agent/src/erniebot_agent/file/base.py +++ b/erniebot-agent/src/erniebot_agent/file/base.py @@ -86,6 +86,7 @@ async def read_contents(self) -> bytes: raise NotImplementedError async def write_contents_to(self, local_path: Union[str, os.PathLike]) -> None: + """Create a file to the location you want. """ contents = await self.read_contents() await anyio.Path(local_path).write_bytes(contents) diff --git a/erniebot-agent/src/erniebot_agent/file/file_manager.py b/erniebot-agent/src/erniebot_agent/file/file_manager.py index 138cc9592..76f7bfcc0 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/file_manager.py @@ -350,7 +350,6 @@ async def list_remote_files(self) -> List[RemoteFile]: files = await self.remote_file_client.list_files() return files -<<<<<<< HEAD def look_up_file_by_id(self, file_id: str) -> Optional[File]: """ Look up a file by its ID. @@ -365,9 +364,6 @@ def look_up_file_by_id(self, file_id: str) -> Optional[File]: FileError: If the file with the specified ID is not found. """ -======= - def look_up_file_by_id(self, file_id: str) -> File: ->>>>>>> d3723952ceac7d4a5ed206dc23b2b0196efa16ca self.ensure_not_closed() file = self._file_registry.look_up_file(file_id) if file is None: @@ -389,6 +385,7 @@ def list_registered_files(self) -> List[File]: return self._file_registry.list_files() async def prune(self) -> None: + """Clean local cache of file manager.""" for file in self._fully_managed_files: if isinstance(file, RemoteFile): # FIXME: Currently this is not supported. @@ -403,6 +400,7 @@ async def prune(self) -> None: self._fully_managed_files.clear() async def close(self) -> None: + """Delete the file manager and clean up its cache""" if not self._closed: if self._remote_file_client is not None: await self._remote_file_client.close() diff --git a/erniebot-agent/src/erniebot_agent/file/local_file.py b/erniebot-agent/src/erniebot_agent/file/local_file.py index f32027be3..d77efe044 100644 --- a/erniebot-agent/src/erniebot_agent/file/local_file.py +++ b/erniebot-agent/src/erniebot_agent/file/local_file.py @@ -27,6 +27,21 @@ def create_local_file_from_path( file_purpose: protocol.FilePurpose, file_metadata: Dict[str, Any], ) -> "LocalFile": + """ + Create a LocalFile object from a local file path. + + Args: + file_path (pathlib.Path): The path to the local file. + file_purpose (protocol.FilePurpose): The purpose or use case of the file. + file_metadata (Dict[str, Any]): Additional metadata associated with the file. + + Returns: + LocalFile: The created LocalFile object. + + Raises: + FileNotFoundError: If the specified file does not exist. + + """ if not file_path.exists(): raise FileNotFoundError(f"File {file_path} does not exist.") file_id = _generate_local_file_id() @@ -46,6 +61,19 @@ def create_local_file_from_path( class LocalFile(File): + """ + Represents a local file. + + Attributes: + path (pathlib.Path): The path to the local file. + + Methods: + __init__: Initialize a LocalFile object. + read_contents: Asynchronously read the contents of the local file. + _get_attrs_str: Helper method to get a string representation of object attributes. + + """ + def __init__( self, *, diff --git a/erniebot-agent/src/erniebot_agent/file/protocol.py b/erniebot-agent/src/erniebot_agent/file/protocol.py index 6caad1424..047dfc311 100644 --- a/erniebot-agent/src/erniebot_agent/file/protocol.py +++ b/erniebot-agent/src/erniebot_agent/file/protocol.py @@ -31,6 +31,7 @@ def create_local_file_id_from_uuid(uuid: str) -> str: + """Create a random local file id.""" return _LOCAL_FILE_ID_PREFIX + uuid @@ -39,26 +40,32 @@ def get_timestamp() -> str: def is_file_id(str_: str) -> bool: + """Judge whether a file id is valid or not.""" return is_local_file_id(str_) or is_remote_file_id(str_) def is_local_file_id(str_: str) -> bool: + """Judge whether a file id is a valid local file id or not.""" return _compiled_local_file_id_pattern.fullmatch(str_) is not None def is_remote_file_id(str_: str) -> bool: + """Judge whether a file id is a valid remote file id or not.""" return _compiled_remote_file_id_pattern.fullmatch(str_) is not None def extract_file_ids(str_: str) -> List[str]: + """Find all file ids in a string.""" return extract_local_file_ids(str_) + extract_remote_file_ids(str_) def extract_local_file_ids(str_: str) -> List[str]: + """Find all local file ids in a string.""" return _compiled_local_file_id_pattern.findall(str_) def extract_remote_file_ids(str_: str) -> List[str]: + """Find all remote file ids in a string.""" return _compiled_remote_file_id_pattern.findall(str_) From 6ab9145e64f23ea8b533d82ac56f528dbd992a68 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 10:23:17 +0800 Subject: [PATCH 34/63] renew docstring v2 --- .../src/erniebot_agent/file/__init__.py | 5 +- .../src/erniebot_agent/file/base.py | 2 +- .../src/erniebot_agent/file/file_manager.py | 32 +----------- .../src/erniebot_agent/file/local_file.py | 31 ++++++++++-- .../src/erniebot_agent/file/remote_file.py | 49 +++++++++++++++++++ 5 files changed, 83 insertions(+), 36 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/file/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py index 53f7d1d0d..d4c09c9ec 100644 --- a/erniebot-agent/src/erniebot_agent/file/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file/__init__.py @@ -11,15 +11,16 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. + """ -File module is used to manage the file system by a global file_manager. +File module is used to manage the file system by a global file_manager. Including `local file` and `remote file`. A few notes about the current state of this submodule: - If you do not set environment variable `EB_ACCESS_TOKEN`, it will be under default setting. - Method `configure_global_file_manager` can only be called once at the beginning. - When you want to get a file manger, you can use method `get_global_file_manager`. -- If you want to get the content of `File` object, you can use `read_contents` +- If you want to get the content of `File` object, you can use `read_contents` and use `write_contents_to` create the file to location you want. """ diff --git a/erniebot-agent/src/erniebot_agent/file/base.py b/erniebot-agent/src/erniebot_agent/file/base.py index 951a11fc9..53b6db88d 100644 --- a/erniebot-agent/src/erniebot_agent/file/base.py +++ b/erniebot-agent/src/erniebot_agent/file/base.py @@ -86,7 +86,7 @@ async def read_contents(self) -> bytes: raise NotImplementedError async def write_contents_to(self, local_path: Union[str, os.PathLike]) -> None: - """Create a file to the location you want. """ + """Create a file to the location you want.""" contents = await self.read_contents() await anyio.Path(local_path).write_bytes(contents) diff --git a/erniebot-agent/src/erniebot_agent/file/file_manager.py b/erniebot-agent/src/erniebot_agent/file/file_manager.py index 675974be8..2d54cc369 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/file_manager.py @@ -48,8 +48,8 @@ FilePath: TypeAlias = Union[str, os.PathLike] -<<<<<<< HEAD -class FileManager(Closeable): +@final +class FileManager(Closeable, Noncopyable): """ Manages files, providing methods for creating, retrieving, and listing files. @@ -68,10 +68,6 @@ class FileManager(Closeable): """ -======= -@final -class FileManager(Closeable, Noncopyable): ->>>>>>> 357a28c240ace535468171bb56d97ca2f454dc58 _temp_dir: Optional[tempfile.TemporaryDirectory] = None def __init__( @@ -110,26 +106,6 @@ def __init__( self._closed = False @property -<<<<<<< HEAD - def remote_file_client(self) -> RemoteFileClient: - """ - Get the remote file client. - - Returns: - RemoteFileClient: The remote file client. - - Raises: - AttributeError: If no remote file client is set. - - """ - if self._remote_file_client is None: - raise AttributeError("No remote file client is set.") - else: - return self._remote_file_client - - @property -======= ->>>>>>> 357a28c240ace535468171bb56d97ca2f454dc58 def closed(self): return self._closed @@ -403,16 +379,12 @@ def list_registered_files(self) -> List[File]: return self._file_registry.list_files() async def prune(self) -> None: -<<<<<<< HEAD """Clean local cache of file manager.""" - for file in self._fully_managed_files: -======= while True: try: file = self._fully_managed_files.pop() except IndexError: break ->>>>>>> 357a28c240ace535468171bb56d97ca2f454dc58 if isinstance(file, RemoteFile): # FIXME: Currently this is not supported. # await file.delete() diff --git a/erniebot-agent/src/erniebot_agent/file/local_file.py b/erniebot-agent/src/erniebot_agent/file/local_file.py index d77efe044..3e80b8784 100644 --- a/erniebot-agent/src/erniebot_agent/file/local_file.py +++ b/erniebot-agent/src/erniebot_agent/file/local_file.py @@ -32,7 +32,8 @@ def create_local_file_from_path( Args: file_path (pathlib.Path): The path to the local file. - file_purpose (protocol.FilePurpose): The purpose or use case of the file. + file_purpose (protocol.FilePurpose): The purpose or use case of the file, + including "assistants" and "assistants_output". file_metadata (Dict[str, Any]): Additional metadata associated with the file. Returns: @@ -65,12 +66,19 @@ class LocalFile(File): Represents a local file. Attributes: + id (str): Unique identifier for the file. + filename (str): File name. + byte_size (int): Size of the file in bytes. + created_at (str): Timestamp indicating the file creation time. + purpose (str): Purpose or use case of the file, + including "assistants" and "assistants_output". + metadata (Dict[str, Any]): Additional metadata associated with the file. path (pathlib.Path): The path to the local file. Methods: - __init__: Initialize a LocalFile object. read_contents: Asynchronously read the contents of the local file. - _get_attrs_str: Helper method to get a string representation of object attributes. + write_contents_to: Asynchronously write the file contents to a local path. + get_file_repr: Return a string representation for use in specific contexts. """ @@ -86,6 +94,23 @@ def __init__( path: pathlib.Path, validate_file_id: bool = True, ) -> None: + """ + Initialize a LocalFile object. + + Args: + id (str): The unique identifier for the file. + filename (str): The name of the file. + byte_size (int): The size of the file in bytes. + created_at (str): The timestamp indicating the file creation time. + purpose (protocol.FilePurpose): The purpose or use case of the file. + metadata (Dict[str, Any]): Additional metadata associated with the file. + path (pathlib.Path): The path to the local file. + validate_file_id (bool): Flag to validate the file ID. Default is True. + + Raises: + ValueError: If the file ID is invalid. + + """ if validate_file_id: if not protocol.is_local_file_id(id): raise ValueError(f"Invalid file ID: {id}") diff --git a/erniebot-agent/src/erniebot_agent/file/remote_file.py b/erniebot-agent/src/erniebot_agent/file/remote_file.py index f9d851445..8c9415283 100644 --- a/erniebot-agent/src/erniebot_agent/file/remote_file.py +++ b/erniebot-agent/src/erniebot_agent/file/remote_file.py @@ -28,6 +28,28 @@ class RemoteFile(File): + """ + Represents a remote file. + + Attributes: + id (str): Unique identifier for the file. + filename (str): File name. + byte_size (int): Size of the file in bytes. + created_at (str): Timestamp indicating the file creation time. + purpose (str): Purpose or use case of the file, + including "assistants" and "assistants_output". + metadata (Dict[str, Any]): Additional metadata associated with the file. + client (RemoteFileClient): The client of remote file. + + Methods: + read_contents: Asynchronously read the contents of the local file. + write_contents_to: Asynchronously write the file contents to a local path. + get_file_repr: Return a string representation for use in specific contexts. + delete: Asynchronously delete the file from client. + create_temporary_url: Asynchronously create a temporary URL for the file. + + """ + def __init__( self, *, @@ -65,6 +87,7 @@ async def delete(self) -> None: await self._client.delete_file(self.id) async def create_temporary_url(self, expire_after: float = 600) -> str: + """To create a temporary valid URL for the file.""" return await self._client.create_temporary_url(self.id, expire_after) def get_file_repr_with_url(self, url: str) -> str: @@ -100,6 +123,20 @@ async def create_temporary_url(self, file_id: str, expire_after: float) -> str: class AIStudioFileClient(RemoteFileClient): + """ + Recommended remote file client: AI Studio. + + Methods: + upload_file: Upload a file to AI Studio client. + retrieve_file: Retrieve information about a file from AI Studio. + retrieve_file_contents: Retrieve the contents of a file from AI Studio. + list_files: List files available in AI Studio. + delete_file: Delete a file in AI Studio client(#TODO: not supported now). + create_temporary_url: Create a temporary URL for accessing a file in AI Studio. + close: Close the AIStudioFileClient. + + """ + _BASE_URL: ClassVar[str] = "https://sandbox-aistudio.baidu.com" _UPLOAD_ENDPOINT: ClassVar[str] = "/llm/lmapp/files" _RETRIEVE_ENDPOINT: ClassVar[str] = "/llm/lmapp/files/{file_id}" @@ -109,6 +146,14 @@ class AIStudioFileClient(RemoteFileClient): def __init__( self, access_token: str, *, aiohttp_session: Optional[aiohttp.ClientSession] = None ) -> None: + """ + Initialize the AIStudioFileClient. + + Args: + access_token (str): The access token for AI Studio. + aiohttp_session (Optional[aiohttp.ClientSession]): A custom aiohttp session (default is None). + + """ super().__init__() self._access_token = access_token if aiohttp_session is None: @@ -123,6 +168,7 @@ def closed(self) -> bool: async def upload_file( self, file_path: pathlib.Path, file_purpose: protocol.FilePurpose, file_metadata: Dict[str, Any] ) -> RemoteFile: + """Upload a file to AI Studio client.""" self.ensure_not_closed() url = self._get_url(self._UPLOAD_ENDPOINT) headers: Dict[str, str] = {} @@ -143,6 +189,7 @@ async def upload_file( return self._create_file_obj_from_dict(result) async def retrieve_file(self, file_id: str) -> RemoteFile: + """Retrieve a file in AI Studio client by id.""" self.ensure_not_closed() url = self._get_url(self._RETRIEVE_ENDPOINT).format(file_id=file_id) headers: Dict[str, str] = {} @@ -157,6 +204,7 @@ async def retrieve_file(self, file_id: str) -> RemoteFile: return self._create_file_obj_from_dict(result) async def retrieve_file_contents(self, file_id: str) -> bytes: + """Retrieve file content in AI Studio client by id.""" self.ensure_not_closed() url = self._get_url(self._RETRIEVE_CONTENTS_ENDPOINT).format(file_id=file_id) headers: Dict[str, str] = {} @@ -170,6 +218,7 @@ async def retrieve_file_contents(self, file_id: str) -> bytes: return resp_bytes async def list_files(self) -> List[RemoteFile]: + """List files in AI Studio client.""" self.ensure_not_closed() url = self._get_url(self._LIST_ENDPOINT) headers: Dict[str, str] = {} From c1d0d947248773915895c35795258c2dbcad8212 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 10:24:52 +0800 Subject: [PATCH 35/63] fix ak definition --- erniebot-agent/src/erniebot_agent/file/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erniebot-agent/src/erniebot_agent/file/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py index d4c09c9ec..df2da17d7 100644 --- a/erniebot-agent/src/erniebot_agent/file/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file/__init__.py @@ -17,7 +17,7 @@ Including `local file` and `remote file`. A few notes about the current state of this submodule: -- If you do not set environment variable `EB_ACCESS_TOKEN`, it will be under default setting. +- If you do not set environment variable `AISTUDIO_ACCESS_TOKEN`, it will be under default setting. - Method `configure_global_file_manager` can only be called once at the beginning. - When you want to get a file manger, you can use method `get_global_file_manager`. - If you want to get the content of `File` object, you can use `read_contents` From d184f6344f5070bfa02ee74512124fb4a0c07868 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 10:56:37 +0800 Subject: [PATCH 36/63] renew docstring file registry --- .../src/erniebot_agent/file/file_registry.py | 99 ++++++++++++++++--- 1 file changed, 87 insertions(+), 12 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/file/file_registry.py b/erniebot-agent/src/erniebot_agent/file/file_registry.py index f3de22a9a..8246c9b45 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_registry.py +++ b/erniebot-agent/src/erniebot_agent/file/file_registry.py @@ -12,32 +12,107 @@ # See the License for the specific language governing permissions and # limitations under the License. -from typing import Dict, List, Optional, final +import threading +from typing import Dict, List, Optional from erniebot_agent.file.base import File +from erniebot_agent.utils.misc import Singleton -@final -class FileRegistry(object): +class FileRegistry(metaclass=Singleton): + """ + Singleton class for managing file registration. + + + Methods: + register_file: Register a file in the registry. + unregister_file: Unregister a file from the registry. + look_up_file: Look up a file by its ID in the registry. + list_files: Get a list of all registered files. + + """ + def __init__(self) -> None: super().__init__() self._id_to_file: Dict[str, File] = {} + self._lock = threading.Lock() def register_file(self, file: File, *, allow_overwrite: bool = False) -> None: + """ + Register a file in the registry. + + Args: + file (File): The file object to register. + allow_overwrite (bool): Allow overwriting if a file with the same ID is already registered. + + Returns: + None + + Raises: + RuntimeError: If the file ID is already registered and allow_overwrite is False. + + """ file_id = file.id - if file_id in self._id_to_file: - if not allow_overwrite: - raise ValueError(f"File with ID {repr(file_id)} is already registered.") - self._id_to_file[file_id] = file + with self._lock: + if not allow_overwrite and file_id in self._id_to_file: + raise RuntimeError(f"ID {repr(file_id)} is already registered.") + self._id_to_file[file_id] = file def unregister_file(self, file: File) -> None: + """ + Unregister a file from the registry. + + Args: + file (File): The file object to unregister. + + Returns: + None + + Raises: + RuntimeError: If the file ID is not registered. + + """ file_id = file.id - if file_id not in self._id_to_file: - raise ValueError(f"File with ID {repr(file_id)} is not registered.") - self._id_to_file.pop(file_id) + with self._lock: + if file_id not in self._id_to_file: + raise RuntimeError(f"ID {repr(file_id)} is not registered.") + self._id_to_file.pop(file_id) def look_up_file(self, file_id: str) -> Optional[File]: - return self._id_to_file.get(file_id, None) + """ + Look up a file by its ID in the registry. + + Args: + file_id (str): The ID of the file to look up. + + Returns: + Optional[File]: The File object if found, or None if not found. + + """ + with self._lock: + return self._id_to_file.get(file_id, None) def list_files(self) -> List[File]: - return list(self._id_to_file.values()) + """ + Get a list of all registered files. + + Returns: + List[File]: The list of registered File objects. + + """ + with self._lock: + return list(self._id_to_file.values()) + + +_file_registry = FileRegistry() + + +def get_file_registry() -> FileRegistry: + """ + Get the global instance of the FileRegistry. + + Returns: + FileRegistry: The global instance of the FileRegistry. + + """ + return _file_registry From 1a204341eec2013c470e71f289f3524aecef544d Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 11:47:56 +0800 Subject: [PATCH 37/63] add file.md v0 --- docs/package/erniebot_agent/file.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) create mode 100644 docs/package/erniebot_agent/file.md diff --git a/docs/package/erniebot_agent/file.md b/docs/package/erniebot_agent/file.md new file mode 100644 index 000000000..3309dc2dc --- /dev/null +++ b/docs/package/erniebot_agent/file.md @@ -0,0 +1,17 @@ +# File Module + +::: erniebot_agent.file + + options: + + summary: true + + members: + + - Memory + + - LocalFile + + - LimitTokensMemory + + - SlidingWindowMemory From c6268d8e946e04f673cb5c0d3ded823f8082b4e1 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 11:59:47 +0800 Subject: [PATCH 38/63] update file.md --- docs/package/erniebot_agent/file.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/package/erniebot_agent/file.md b/docs/package/erniebot_agent/file.md index 3309dc2dc..35e6c1109 100644 --- a/docs/package/erniebot_agent/file.md +++ b/docs/package/erniebot_agent/file.md @@ -8,10 +8,8 @@ members: - - Memory + - FileManager - LocalFile - - LimitTokensMemory - - - SlidingWindowMemory + - RemoteFile From 7d7e87ea0a618dfe4656ed1826e9d4dc8fa66b7d Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 13:52:15 +0800 Subject: [PATCH 39/63] add detailed des --- docs/modules/file.md | 63 +++++++++++++++++++ docs/package/erniebot_agent/file.md | 18 +++--- .../src/erniebot_agent/file/README.md | 63 +++++++++++++++++++ .../src/erniebot_agent/file/__init__.py | 8 +++ mkdocs.yml | 2 + 5 files changed, 143 insertions(+), 11 deletions(-) create mode 100644 docs/modules/file.md create mode 100644 erniebot-agent/src/erniebot_agent/file/README.md diff --git a/docs/modules/file.md b/docs/modules/file.md new file mode 100644 index 000000000..b3a97114f --- /dev/null +++ b/docs/modules/file.md @@ -0,0 +1,63 @@ +# File 模块介绍 + +## 1. File 简介 + +在建立一个Agent应用的过程中，由于LLM本身是无状态的，因此很重要的一点就是赋予Agent记忆能力。Agent的记忆能力主要可以分为长期记忆和短期记忆。 + +* 长期记忆通过文件/数据库的形式存储，是不会被遗忘的内容，每次判断需要相关知识就可以retrieval的方式，找到最相关的内容放入消息帮助LLM分析得到结果。 +* 短期记忆则是直接在消息中体现，是LLM接触的一手信息，但是也受限于上下文窗口，更容易被遗忘。 + +这里我们简述在我们记录短期记忆的方式，即在Memory模块中存储消息，并通过消息裁剪，控制消息总条数不会超过上下文窗口。 + +在使用层面，Memory将传入Agent类中，用于记录多轮的消息，即Agent会在每一轮对话中和Memory有一次交互：即在LLM产生最终结论之后，将第一条HumanMessage和最后一条AIMessage加入到Memory中。 + +## 2. File 基类介绍 + +`File` 类是文件管理模块的基础类，用于表示通用的文件对象。它包含文件的基本属性，如文件ID、文件名、文件大小、创建时间、文件用途和文件元数据。此外，`File` 类还定义了一系列抽象方法，比较常用的有：异步读取文件内容的 `read_contents` 方法，以及将文件内容写入本地路径的 `write_contents_to` 方法以及一些辅助方法：生成文件的字符串表示形式、转换为字典形式等。在File类的内部，其主要有两个继承子类，一个是 `Local File`，一个是 `Remote File`。以下是 `File` 基类的属性以及方法介绍。 + +| 属性 | 类型 | 描述 | +| ---------- | -------------- | --------------------------------------------------------- | +| id | str | 文件的唯一标识符 | +| filename | str | 文件名 | +| byte_size | int | 文件大小（以字节为单位） | +| created_at | str | 文件创建时间的时间戳 | +| purpose | str | 文件的目的或用途，有"assistants", "assistants_output"两种 | +| metadata | Dict[str, Any] | 与文件相关的附加元数据 | + +| 方法 | 描述 | +| ----------------- | ------------------------------ | +| read_contents | 异步读取文件内容 | +| write_contents_to | 异步将文件内容写入本地路径 | +| get_file_repr | 返回用于特定上下文的字符串表示 | +| to_dict | 将File对象转换为字典 | + +### 2.2 File 子类 + +#### 2.2.1 LocalFile 类 + +`LocalFile` 是 `File` 的子类，表示本地文件。除了继承自基类的属性外，它还添加了文件路径属性 `path`，用于表示文件在本地文件系统中的路径。 + +#### 2.2.2 RemoteFile 类 + +`RemoteFile` 也是 `File` 的子类，表示远程文件。它与 `LocalFile` 不同之处在于，它的文件内容存储在远程文件服务器交。`RemoteFile` 类还包含与远程文件服务器交互的相关逻辑。它还添加了文件服务器属性 `client`，用于表示文件的服务器。 + + +## 3. FileManager 类介绍 + +`FileManager` 类是一个高级文件管理工具，封装了文件的创建、上传、删除等高级操作。它可能依赖于 `FileRegistry` 来管理文件的一致性和唯一性。`FileManager` 还包含了与远程文件服务器交互的逻辑，通过 `RemoteFileClient` 完成上传、下载、删除等文件操作。 + +## 4. RemoteFileClient 类介绍 + +`RemoteFileClient` 是用于与远程文件服务器交互的类。它定义了文件上传、文件下载、文件删除等操作的方法。`AIStudioFileClient` 是 `RemoteFileClient` 的一个具体实现，用于与 AI Studio 文件服务交互。 + +## 5. FileRegistry 类介绍 + +`FileRegistry` 是一个单例类，用于在整个应用程序中管理文件的注册和查找。它提供了方法来注册、注销、查找和列举已注册的文件。`FileRegistry` 通过全局唯一的实例 `_file_registry` 来跟踪应用程序中的所有文件。 + +## 6. 使用方法 + +1. 创建 `File` 对象，可以选择使用 `LocalFile` 或 `RemoteFile` 的子类。 +2. 使用 `FileRegistry` 注册文件，确保文件的唯一性。 +3. 使用 `FileManager` 进行高级文件操作，如上传、下载、删除等。 + +通过这一系列的类，文件管理模块提供了灵活、高效的文件管理解决方案，适用于本地文件和远程文件的处理。 diff --git a/docs/package/erniebot_agent/file.md b/docs/package/erniebot_agent/file.md index 35e6c1109..be3168242 100644 --- a/docs/package/erniebot_agent/file.md +++ b/docs/package/erniebot_agent/file.md @@ -1,15 +1,11 @@ # File Module -::: erniebot_agent.file +::: erniebot_agent.file options: - - summary: true - - members: - - - FileManager - - - LocalFile - - - RemoteFile + summary: true + members: + - File + - LocalFile + - RemoteFile + - FileManager diff --git a/erniebot-agent/src/erniebot_agent/file/README.md b/erniebot-agent/src/erniebot_agent/file/README.md new file mode 100644 index 000000000..b3a97114f --- /dev/null +++ b/erniebot-agent/src/erniebot_agent/file/README.md @@ -0,0 +1,63 @@ +# File 模块介绍 + +## 1. File 简介 + +在建立一个Agent应用的过程中，由于LLM本身是无状态的，因此很重要的一点就是赋予Agent记忆能力。Agent的记忆能力主要可以分为长期记忆和短期记忆。 + +* 长期记忆通过文件/数据库的形式存储，是不会被遗忘的内容，每次判断需要相关知识就可以retrieval的方式，找到最相关的内容放入消息帮助LLM分析得到结果。 +* 短期记忆则是直接在消息中体现，是LLM接触的一手信息，但是也受限于上下文窗口，更容易被遗忘。 + +这里我们简述在我们记录短期记忆的方式，即在Memory模块中存储消息，并通过消息裁剪，控制消息总条数不会超过上下文窗口。 + +在使用层面，Memory将传入Agent类中，用于记录多轮的消息，即Agent会在每一轮对话中和Memory有一次交互：即在LLM产生最终结论之后，将第一条HumanMessage和最后一条AIMessage加入到Memory中。 + +## 2. File 基类介绍 + +`File` 类是文件管理模块的基础类，用于表示通用的文件对象。它包含文件的基本属性，如文件ID、文件名、文件大小、创建时间、文件用途和文件元数据。此外，`File` 类还定义了一系列抽象方法，比较常用的有：异步读取文件内容的 `read_contents` 方法，以及将文件内容写入本地路径的 `write_contents_to` 方法以及一些辅助方法：生成文件的字符串表示形式、转换为字典形式等。在File类的内部，其主要有两个继承子类，一个是 `Local File`，一个是 `Remote File`。以下是 `File` 基类的属性以及方法介绍。 + +| 属性 | 类型 | 描述 | +| ---------- | -------------- | --------------------------------------------------------- | +| id | str | 文件的唯一标识符 | +| filename | str | 文件名 | +| byte_size | int | 文件大小（以字节为单位） | +| created_at | str | 文件创建时间的时间戳 | +| purpose | str | 文件的目的或用途，有"assistants", "assistants_output"两种 | +| metadata | Dict[str, Any] | 与文件相关的附加元数据 | + +| 方法 | 描述 | +| ----------------- | ------------------------------ | +| read_contents | 异步读取文件内容 | +| write_contents_to | 异步将文件内容写入本地路径 | +| get_file_repr | 返回用于特定上下文的字符串表示 | +| to_dict | 将File对象转换为字典 | + +### 2.2 File 子类 + +#### 2.2.1 LocalFile 类 + +`LocalFile` 是 `File` 的子类，表示本地文件。除了继承自基类的属性外，它还添加了文件路径属性 `path`，用于表示文件在本地文件系统中的路径。 + +#### 2.2.2 RemoteFile 类 + +`RemoteFile` 也是 `File` 的子类，表示远程文件。它与 `LocalFile` 不同之处在于，它的文件内容存储在远程文件服务器交。`RemoteFile` 类还包含与远程文件服务器交互的相关逻辑。它还添加了文件服务器属性 `client`，用于表示文件的服务器。 + + +## 3. FileManager 类介绍 + +`FileManager` 类是一个高级文件管理工具，封装了文件的创建、上传、删除等高级操作。它可能依赖于 `FileRegistry` 来管理文件的一致性和唯一性。`FileManager` 还包含了与远程文件服务器交互的逻辑，通过 `RemoteFileClient` 完成上传、下载、删除等文件操作。 + +## 4. RemoteFileClient 类介绍 + +`RemoteFileClient` 是用于与远程文件服务器交互的类。它定义了文件上传、文件下载、文件删除等操作的方法。`AIStudioFileClient` 是 `RemoteFileClient` 的一个具体实现，用于与 AI Studio 文件服务交互。 + +## 5. FileRegistry 类介绍 + +`FileRegistry` 是一个单例类，用于在整个应用程序中管理文件的注册和查找。它提供了方法来注册、注销、查找和列举已注册的文件。`FileRegistry` 通过全局唯一的实例 `_file_registry` 来跟踪应用程序中的所有文件。 + +## 6. 使用方法 + +1. 创建 `File` 对象，可以选择使用 `LocalFile` 或 `RemoteFile` 的子类。 +2. 使用 `FileRegistry` 注册文件，确保文件的唯一性。 +3. 使用 `FileManager` 进行高级文件操作，如上传、下载、删除等。 + +通过这一系列的类，文件管理模块提供了灵活、高效的文件管理解决方案，适用于本地文件和远程文件的处理。 diff --git a/erniebot-agent/src/erniebot_agent/file/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py index df2da17d7..9751c8e89 100644 --- a/erniebot-agent/src/erniebot_agent/file/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file/__init__.py @@ -17,11 +17,19 @@ Including `local file` and `remote file`. A few notes about the current state of this submodule: + - If you do not set environment variable `AISTUDIO_ACCESS_TOKEN`, it will be under default setting. + - Method `configure_global_file_manager` can only be called once at the beginning. + - When you want to get a file manger, you can use method `get_global_file_manager`. + - If you want to get the content of `File` object, you can use `read_contents` and use `write_contents_to` create the file to location you want. """ from erniebot_agent.file.global_file_manager_handler import GlobalFileManagerHandler +from erniebot_agent.file.base import File +from erniebot_agent.file.file_manager import FileManager +from erniebot_agent.file.local_file import LocalFile +from erniebot_agent.file.remote_file import RemoteFile \ No newline at end of file diff --git a/mkdocs.yml b/mkdocs.yml index 242dfc962..aaa6571d2 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -9,10 +9,12 @@ nav: - 介绍: 'tutorials/index.md' - 核心模块: - memory: 'modules/memory.md' + - file: 'modules/file.md' - API: - erniebot-agent: - agents: "package/erniebot_agent/agents.md" - memory: "package/erniebot_agent/memory.md" + - file: "package/erniebot_agent/file.md" - chat_models: "package/erniebot_agent/chat_models.md" theme: From c3c194a483a8faf4b4e28ee80a2bab7a4cdbd147 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 16:02:19 +0800 Subject: [PATCH 40/63] renew description of file manager and global file manager --- docs/modules/file.md | 84 ++++++++++++++----- docs/package/erniebot_agent/file.md | 34 +++++++- .../src/erniebot_agent/file/README.md | 74 +++++++++++----- .../src/erniebot_agent/file/__init__.py | 27 ++++-- .../src/erniebot_agent/file/file_manager.py | 12 ++- .../file/global_file_manager_handler.py | 60 +++++++++++++ 6 files changed, 235 insertions(+), 56 deletions(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index b3a97114f..4d76253f8 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -1,19 +1,20 @@ + # File 模块介绍 ## 1. File 简介 -在建立一个Agent应用的过程中，由于LLM本身是无状态的，因此很重要的一点就是赋予Agent记忆能力。Agent的记忆能力主要可以分为长期记忆和短期记忆。 - -* 长期记忆通过文件/数据库的形式存储，是不会被遗忘的内容，每次判断需要相关知识就可以retrieval的方式，找到最相关的内容放入消息帮助LLM分析得到结果。 -* 短期记忆则是直接在消息中体现，是LLM接触的一手信息，但是也受限于上下文窗口，更容易被遗忘。 - -这里我们简述在我们记录短期记忆的方式，即在Memory模块中存储消息，并通过消息裁剪，控制消息总条数不会超过上下文窗口。 - -在使用层面，Memory将传入Agent类中，用于记录多轮的消息，即Agent会在每一轮对话中和Memory有一次交互：即在LLM产生最终结论之后，将第一条HumanMessage和最后一条AIMessage加入到Memory中。 +文件管理模块提供了用于管理文件的一系列类，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。 ## 2. File 基类介绍 -`File` 类是文件管理模块的基础类，用于表示通用的文件对象。它包含文件的基本属性，如文件ID、文件名、文件大小、创建时间、文件用途和文件元数据。此外，`File` 类还定义了一系列抽象方法，比较常用的有：异步读取文件内容的 `read_contents` 方法，以及将文件内容写入本地路径的 `write_contents_to` 方法以及一些辅助方法：生成文件的字符串表示形式、转换为字典形式等。在File类的内部，其主要有两个继承子类，一个是 `Local File`，一个是 `Remote File`。以下是 `File` 基类的属性以及方法介绍。 +`File` 类是文件管理模块的基础类，用于表示通用的文件对象（不建议自行创建 `File` 类以免无法被 `Agent`识别）。它包含文件的基本属性，如文件ID、文件名、文件大小、创建时间、文件用途和文件元数据。此外， `File `类还定义了一系列抽象方法，比较常用的有：异步读取文件内容的 `read_contents `方法，以及将文件内容写入本地路径的 `write_contents_to `方法以及一些辅助方法：生成文件的字符串表示形式、转换为字典形式等。在File类的内部，其主要有两个继承子类，一个是 `Local File `，一个是 `Remote File `。以下是 `File` 基类的属性以及方法介绍。 | 属性 | 类型 | 描述 | | ---------- | -------------- | --------------------------------------------------------- | @@ -39,25 +40,66 @@ #### 2.2.2 RemoteFile 类 -`RemoteFile` 也是 `File` 的子类，表示远程文件。它与 `LocalFile` 不同之处在于，它的文件内容存储在远程文件服务器交。`RemoteFile` 类还包含与远程文件服务器交互的相关逻辑。它还添加了文件服务器属性 `client`，用于表示文件的服务器。 - +`RemoteFile` 也是 `File` 的子类，表示远程文件。它与 `LocalFile` 不同之处在于，它的文件内容存储在远程文件服务器交。`RemoteFile` 类还包含与远程文件服务器交互的相关逻辑。 ## 3. FileManager 类介绍 -`FileManager` 类是一个高级文件管理工具，封装了文件的创建、上传、删除等高级操作。它可能依赖于 `FileRegistry` 来管理文件的一致性和唯一性。`FileManager` 还包含了与远程文件服务器交互的逻辑，通过 `RemoteFileClient` 完成上传、下载、删除等文件操作。 +`FileManager` 类是一个高级文件管理工具，封装了文件的创建、上传、删除等高级操作，以及与 `Agent`进行交互，无论是 `Local File `还是 `RemoteFIle `都可以使用它来统一管理。`FileManager `集成了与远程文件服务器交互的逻辑，通过 `RemoteFileClient `完成上传、下载、删除等文件操作以及与本地文件交互的逻辑，从本地地址创建 `Local File `。它依赖于 `FileRegistry` 来对文件进行用于在整个应用程序中管理文件的注册和查找。 -## 4. RemoteFileClient 类介绍 +以下是相关的属性和方法 -`RemoteFileClient` 是用于与远程文件服务器交互的类。它定义了文件上传、文件下载、文件删除等操作的方法。`AIStudioFileClient` 是 `RemoteFileClient` 的一个具体实现，用于与 AI Studio 文件服务交互。 +| 属性 | 类型 | 描述 | +| ------------------ | ------------------ | ------------------------ | +| remote_file_client | RemoteFileClient | 远程文件客户端。 | +| save_dir | Optional[FilePath] | 用于保存本地文件的目录。 | +| closed | bool | 文件管理器是否已关闭。 | -## 5. FileRegistry 类介绍 +| 方法 | 描述 | +| ---------------------------- | ------------------------------------ | +| create_file_from_path | 从指定文件路径创建文件 | +| create_local_file_from_path | 从文件路径创建本地文件 | +| create_remote_file_from_path | 从文件路径创建远程文件并上传至服务器 | +| create_file_from_bytes | 从字节创建文件 | +| retrieve_remote_file_by_id | 通过ID获取远程文件 | +| look_up_file_by_id | 通过ID查找本地文件 | +| list_remote_files | 列出远程文件 | -`FileRegistry` 是一个单例类，用于在整个应用程序中管理文件的注册和查找。它提供了方法来注册、注销、查找和列举已注册的文件。`FileRegistry` 通过全局唯一的实例 `_file_registry` 来跟踪应用程序中的所有文件。 +注： -## 6. 使用方法 +* `FileManager` 类不可被复制以免造成资源泄露。 +* 如果未指定 `save_dir`，那么当 `FileManager`关闭时，所有与之关联的本地文件都会被回收。 +* 如果 `FileManager` 类有相关联的 `RemoteFileClient`，那么当 `FileManager`关闭时，相关联的 `RemoteFileClient`也会一起关闭。 -1. 创建 `File` 对象，可以选择使用 `LocalFile` 或 `RemoteFile` 的子类。 -2. 使用 `FileRegistry` 注册文件，确保文件的唯一性。 -3. 使用 `FileManager` 进行高级文件操作，如上传、下载、删除等。 +## 4. RemoteFileClient 类介绍 -通过这一系列的类，文件管理模块提供了灵活、高效的文件管理解决方案，适用于本地文件和远程文件的处理。 +`RemoteFileClient` 是用于与远程文件服务器交互的类。它定义了文件上传、文件下载、文件删除等操作的方法。`AIStudioFileClient` 是 `RemoteFileClient` 的一个具体推荐实现，用于与文件服务交互，用户使用 `access token`作为参数用于身份验证，之后能够在AIStudio文件服务中上传、检索、列出文件，以及创建临时URL以访问文件。`RemoteFileClient`使用时被 `FileManager`持有，一旦 `FileManager`关闭，`RemoteFileClient`也会相应被关闭，其中的资源也会被相应释放。 + +## 5. 使用方法 + +1. 通过 `GlobalFileManagerHandler`获取全局的FileManager，通过它来控制所有文件，注：它的生命周期同整个事件循环。 + +```python +from erniebot_agent.file import GlobalFileManagerHandler +async def demo_function(): + file_manager = await GlobalFileManagerHandler().get() +``` + +2. 通过 `GlobalFileManagerHandler`创建 `File` + +```python +from erniebot_agent.file import GlobalFileManagerHandler +async def demo_function(): + file_manager = await GlobalFileManagerHandler().get() + # 从路径创建File, file_type可选择local或者remote file_purpose='assistant'代表用于给LLM输入使用 + local_file = await file_manager.create_file_from_path(file_path='your_path', file_type='local') +``` + +3. 通过 `GlobalFileManagerHandler`搜索 `File` + +```python +from erniebot_agent.file import GlobalFileManagerHandler +async def demo_function(): + file_manager = await GlobalFileManagerHandler().get() + file = file_manager.look_up_file_by_id(file_id='your_file_id') + file_content = await file.read_contents() +``` diff --git a/docs/package/erniebot_agent/file.md b/docs/package/erniebot_agent/file.md index be3168242..d8fe7c467 100644 --- a/docs/package/erniebot_agent/file.md +++ b/docs/package/erniebot_agent/file.md @@ -1,11 +1,43 @@ # File Module - ::: erniebot_agent.file options: summary: true + + +::: erniebot_agent.file.base + options: + summary: true members: - File + + +::: erniebot_agent.file.local_file + options: + summary: true + members: - LocalFile + +::: erniebot_agent.file.remote_file + options: + summary: true + members: - RemoteFile + +::: erniebot_agent.file.file_manager + options: + summary: true + members: - FileManager + +::: erniebot_agent.file.global_file_manager_handler + options: + summary: true + members: + - GlobalFileManagerHandler + +::: erniebot_agent.file.remote_file + options: + summary: true + members: + - AIStudioFileClient \ No newline at end of file diff --git a/erniebot-agent/src/erniebot_agent/file/README.md b/erniebot-agent/src/erniebot_agent/file/README.md index b3a97114f..e07a6c742 100644 --- a/erniebot-agent/src/erniebot_agent/file/README.md +++ b/erniebot-agent/src/erniebot_agent/file/README.md @@ -2,18 +2,11 @@ ## 1. File 简介 -在建立一个Agent应用的过程中，由于LLM本身是无状态的，因此很重要的一点就是赋予Agent记忆能力。Agent的记忆能力主要可以分为长期记忆和短期记忆。 - -* 长期记忆通过文件/数据库的形式存储，是不会被遗忘的内容，每次判断需要相关知识就可以retrieval的方式，找到最相关的内容放入消息帮助LLM分析得到结果。 -* 短期记忆则是直接在消息中体现，是LLM接触的一手信息，但是也受限于上下文窗口，更容易被遗忘。 - -这里我们简述在我们记录短期记忆的方式，即在Memory模块中存储消息，并通过消息裁剪，控制消息总条数不会超过上下文窗口。 - -在使用层面，Memory将传入Agent类中，用于记录多轮的消息，即Agent会在每一轮对话中和Memory有一次交互：即在LLM产生最终结论之后，将第一条HumanMessage和最后一条AIMessage加入到Memory中。 +文件管理模块提供了用于管理文件的一系列类，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。 ## 2. File 基类介绍 -`File` 类是文件管理模块的基础类，用于表示通用的文件对象。它包含文件的基本属性，如文件ID、文件名、文件大小、创建时间、文件用途和文件元数据。此外，`File` 类还定义了一系列抽象方法，比较常用的有：异步读取文件内容的 `read_contents` 方法，以及将文件内容写入本地路径的 `write_contents_to` 方法以及一些辅助方法：生成文件的字符串表示形式、转换为字典形式等。在File类的内部，其主要有两个继承子类，一个是 `Local File`，一个是 `Remote File`。以下是 `File` 基类的属性以及方法介绍。 +`File` 类是文件管理模块的基础类，用于表示通用的文件对象（不建议自行创建 `File` 类以免无法被 `Agent`识别）。它包含文件的基本属性，如文件ID、文件名、文件大小、创建时间、文件用途和文件元数据。此外， `File `类还定义了一系列抽象方法，比较常用的有：异步读取文件内容的 `read_contents `方法，以及将文件内容写入本地路径的 `write_contents_to `方法以及一些辅助方法：生成文件的字符串表示形式、转换为字典形式等。在File类的内部，其主要有两个继承子类，一个是 `Local File `，一个是 `Remote File `。以下是 `File` 基类的属性以及方法介绍。 | 属性 | 类型 | 描述 | | ---------- | -------------- | --------------------------------------------------------- | @@ -39,25 +32,64 @@ #### 2.2.2 RemoteFile 类 -`RemoteFile` 也是 `File` 的子类，表示远程文件。它与 `LocalFile` 不同之处在于，它的文件内容存储在远程文件服务器交。`RemoteFile` 类还包含与远程文件服务器交互的相关逻辑。它还添加了文件服务器属性 `client`，用于表示文件的服务器。 - +`RemoteFile` 也是 `File` 的子类，表示远程文件。它与 `LocalFile` 不同之处在于，它的文件内容存储在远程文件服务器交。`RemoteFile` 类还包含与远程文件服务器交互的相关逻辑。 ## 3. FileManager 类介绍 -`FileManager` 类是一个高级文件管理工具，封装了文件的创建、上传、删除等高级操作。它可能依赖于 `FileRegistry` 来管理文件的一致性和唯一性。`FileManager` 还包含了与远程文件服务器交互的逻辑，通过 `RemoteFileClient` 完成上传、下载、删除等文件操作。 +`FileManager` 类是一个高级文件管理工具，封装了文件的创建、上传、删除等高级操作，以及与 `Agent`进行交互，无论是 `Local File `还是 `RemoteFIle `都可以使用它来统一管理。`FileManager `集成了与远程文件服务器交互的逻辑，通过 `RemoteFileClient `完成上传、下载、删除等文件操作以及与本地文件交互的逻辑，从本地地址创建 `Local File `。它依赖于 `FileRegistry` 来对文件进行用于在整个应用程序中管理文件的注册和查找。 -## 4. RemoteFileClient 类介绍 +以下是相关的属性和方法 -`RemoteFileClient` 是用于与远程文件服务器交互的类。它定义了文件上传、文件下载、文件删除等操作的方法。`AIStudioFileClient` 是 `RemoteFileClient` 的一个具体实现，用于与 AI Studio 文件服务交互。 +| 属性 | 类型 | 描述 | +| ------------------ | ------------------ | ------------------------ | +| remote_file_client | RemoteFileClient | 远程文件客户端。 | +| save_dir | Optional[FilePath] | 用于保存本地文件的目录。 | +| closed | bool | 文件管理器是否已关闭。 | -## 5. FileRegistry 类介绍 +| 方法 | 描述 | +| ---------------------------- | ------------------------------------ | +| create_file_from_path | 从指定文件路径创建文件 | +| create_local_file_from_path | 从文件路径创建本地文件 | +| create_remote_file_from_path | 从文件路径创建远程文件并上传至服务器 | +| create_file_from_bytes | 从字节创建文件 | +| retrieve_remote_file_by_id | 通过ID获取远程文件 | +| look_up_file_by_id | 通过ID查找本地文件 | +| list_remote_files | 列出远程文件 | -`FileRegistry` 是一个单例类，用于在整个应用程序中管理文件的注册和查找。它提供了方法来注册、注销、查找和列举已注册的文件。`FileRegistry` 通过全局唯一的实例 `_file_registry` 来跟踪应用程序中的所有文件。 +注： -## 6. 使用方法 +* `FileManager` 类不可被复制以免造成资源泄露。 +* 如果未指定 `save_dir`，那么当 `FileManager`关闭时，所有与之关联的本地文件都会被回收。 +* 如果 `FileManager` 类有相关联的 `RemoteFileClient`，那么当 `FileManager`关闭时，相关联的 `RemoteFileClient`也会一起关闭。 + +## 4. RemoteFileClient 类介绍 -1. 创建 `File` 对象，可以选择使用 `LocalFile` 或 `RemoteFile` 的子类。 -2. 使用 `FileRegistry` 注册文件，确保文件的唯一性。 -3. 使用 `FileManager` 进行高级文件操作，如上传、下载、删除等。 +`RemoteFileClient` 是用于与远程文件服务器交互的类。它定义了文件上传、文件下载、文件删除等操作的方法。`AIStudioFileClient` 是 `RemoteFileClient` 的一个具体推荐实现，用于与文件服务交互，用户使用 `access token`作为参数用于身份验证，之后能够在AIStudio文件服务中上传、检索、列出文件，以及创建临时URL以访问文件。`RemoteFileClient`使用时被 `FileManager`持有，一旦 `FileManager`关闭，`RemoteFileClient`也会相应被关闭，其中的资源也会被相应释放。 + +## 6. 使用方法 -通过这一系列的类，文件管理模块提供了灵活、高效的文件管理解决方案，适用于本地文件和远程文件的处理。 +1. 通过 `GlobalFileManagerHandler`获取全局的FileManager，通过它来控制所有文件，注：它的生命周期同整个事件循环。 + + ```python + from erniebot_agent.file import GlobalFileManagerHandler + async def demo_function(): + file_manager = await GlobalFileManagerHandler().get() + ``` +2. 通过 `GlobalFileManagerHandler`创建 `File` + + ```python + from erniebot_agent.file import GlobalFileManagerHandler + async def demo_function(): + file_manager = await GlobalFileManagerHandler().get() + # 从路径创建File, file_type可选择local或者remote file_purpose='assistant'代表用于给LLM输入使用 + local_file = await file_manager.create_file_from_path(file_path='your_path', file_type='local') + ``` +3. 通过 `GlobalFileManagerHandler`搜索 `File` + + ```python + from erniebot_agent.file import GlobalFileManagerHandler + async def demo_function(): + file_manager = await GlobalFileManagerHandler().get() + file = file_manager.look_up_file_by_id(file_id='your_file_id') + file_content = await file.read_contents() + ``` diff --git a/erniebot-agent/src/erniebot_agent/file/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py index 9751c8e89..53c3eeb94 100644 --- a/erniebot-agent/src/erniebot_agent/file/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file/__init__.py @@ -13,23 +13,32 @@ # limitations under the License. """ -File module is used to manage the file system by a global file_manager. -Including `local file` and `remote file`. +File module is used to manage the file system by a global file manager: `GlobalFileManagerHandler`. + +We use it to manage `File` obeject, including `LocalFile` and `RemoteFile`. A few notes about the current state of this submodule: - If you do not set environment variable `AISTUDIO_ACCESS_TOKEN`, it will be under default setting. -- Method `configure_global_file_manager` can only be called once at the beginning. +- Method `GlobalFileManagerHandler().configure()` can only be called **once** at the beginning. + +- When you want to get a file manger, you can use method `GlobalFileManagerHandler().get()`. + +- The lifecycle of the `FileManager` class is synchronized with the event loop. -- When you want to get a file manger, you can use method `get_global_file_manager`. +- `FileManager` class is Noncopyable. - If you want to get the content of `File` object, you can use `read_contents` and use `write_contents_to` create the file to location you want. + +- We do **not** recommend you to create `File` object yourself. + +Examples: + >>> from erniebot_agent.file import GlobalFileManagerHandler + >>> async def demo_function(): + >>> file_manager = await GlobalFileManagerHandler().get() """ -from erniebot_agent.file.global_file_manager_handler import GlobalFileManagerHandler -from erniebot_agent.file.base import File -from erniebot_agent.file.file_manager import FileManager -from erniebot_agent.file.local_file import LocalFile -from erniebot_agent.file.remote_file import RemoteFile \ No newline at end of file +from .global_file_manager_handler import GlobalFileManagerHandler +from .remote_file import AIStudioFileClient diff --git a/erniebot-agent/src/erniebot_agent/file/file_manager.py b/erniebot-agent/src/erniebot_agent/file/file_manager.py index 2d54cc369..baa6ab707 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/file_manager.py @@ -56,6 +56,7 @@ class FileManager(Closeable, Noncopyable): Attributes: remote_file_client(RemoteFileClient): The remote file client. save_dir (Optional[FilePath]): Directory for saving local files. + closed: Whether the file manager is closed. Methods: create_file_from_path: Create a file from a specified file path. @@ -156,7 +157,8 @@ async def create_file_from_path( Args: file_path (FilePath): The path to the file. - file_purpose (FilePurpose): The purpose or use case of the file. + file_purpose (FilePurpose): The purpose or use case of the file, + including `assistant`: used for llm and `assistant_output`: used for output. file_metadata (Optional[Dict[str, Any]]): Additional metadata associated with the file. file_type (Optional[Literal["local", "remote"]]): The type of file ("local" or "remote"). @@ -190,7 +192,8 @@ async def create_local_file_from_path( Args: file_path (FilePath): The path to the file. - file_purpose (FilePurpose): The purpose or use case of the file. + file_purpose (FilePurpose): The purpose or use case of the file, + including `assistant`: used for llm and `assistant_output`: used for output. file_metadata (Optional[Dict[str, Any]]): Additional metadata associated with the file. Returns: @@ -212,11 +215,12 @@ async def create_remote_file_from_path( file_metadata: Optional[Dict[str, Any]], ) -> RemoteFile: """ - Create a remote file from a file path. + Create a remote file from a file path and upload it to the client. Args: file_path (FilePath): The path to the file. - file_purpose (FilePurpose): The purpose or use case of the file. + file_purpose (FilePurpose): The purpose or use case of the file, + including `assistant`: used for llm and `assistant_output`: used for output. file_metadata (Optional[Dict[str, Any]]): Additional metadata associated with the file. Returns: diff --git a/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py b/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py index d4d809276..9d654c0d6 100644 --- a/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py +++ b/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py @@ -25,6 +25,18 @@ @final class GlobalFileManagerHandler(Singleton): + """Singleton handler for managing the global FileManager instance. + + This class provides a singleton instance for managing the global FileManager + and allows for its configuration and retrieval. + + + Attributes: + _file_manager (Optional[FileManager]): The global FileManager instance. + _lock (asyncio.Lock): A lock for ensuring thread-safe access to the FileManager. + + """ + _file_manager: Optional[FileManager] def __init__(self) -> None: @@ -33,6 +45,18 @@ def __init__(self) -> None: self._file_manager = None async def get(self) -> FileManager: + """ + Retrieve the global FileManager instance. + + This method returns the existing global FileManager instance, + creating one if it doesn't exist. + + + Returns: + FileManager: The global FileManager instance. + + """ + async with self._lock: if self._file_manager is None: self._file_manager = await self._create_default_file_manager( @@ -50,6 +74,25 @@ async def configure( enable_remote_file: bool = False, **opts: Any, ) -> None: + """ + Configure the global FileManager. + + This method configures the global FileManager with the provided parameters. + If the global FileManager is already set, it raises an error. + + Args: + access_token (Optional[str]): The access token for remote file management. + save_dir (Optional[str]): The directory for saving local files. + enable_remote_file (bool): Whether to enable remote file management. + **opts (Any): Additional options for FileManager. + + Returns: + None + + Raises: + RuntimeError: If the global FileManager is already set. + + """ async with self._lock: if self._file_manager is not None: self._raise_file_manager_already_set_error() @@ -61,6 +104,21 @@ async def configure( ) async def set(self, file_manager: FileManager) -> None: + """ + Set the global FileManager explicitly. + + This method sets the global FileManager instance explicitly. + If the global FileManager is already set, it raises an error. + + Args: + file_manager (FileManager): The FileManager instance to set as global. + + Returns: + None + + Raises: + RuntimeError: If the global FileManager is already set. + """ async with self._lock: if self._file_manager is not None: self._raise_file_manager_already_set_error() @@ -73,6 +131,8 @@ async def _create_default_file_manager( enable_remote_file: bool, **opts: Any, ) -> FileManager: + """Create the default FileManager instance.""" + async def _close_file_manager(): await file_manager.close() From aeafc0407c38ffc5e2015fabb8130d185c171e05 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 16:12:48 +0800 Subject: [PATCH 41/63] debug mypy --- .../src/erniebot_agent/file/file_manager.py | 10 +++++ .../src/erniebot_agent/file/file_registry.py | 43 ++++++------------- 2 files changed, 22 insertions(+), 31 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/file/file_manager.py b/erniebot-agent/src/erniebot_agent/file/file_manager.py index baa6ab707..96b5f2e53 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/file_manager.py @@ -144,6 +144,16 @@ async def create_file_from_path( ... @overload + async def create_file_from_path( + self, + file_path: FilePath, + *, + file_purpose: protocol.FilePurpose = ..., + file_metadata: Optional[Dict[str, Any]] = ..., + file_type: None = ..., + ) -> Union[LocalFile, RemoteFile]: + ... + async def create_file_from_path( self, file_path: FilePath, diff --git a/erniebot-agent/src/erniebot_agent/file/file_registry.py b/erniebot-agent/src/erniebot_agent/file/file_registry.py index 8246c9b45..8feffe3dd 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_registry.py +++ b/erniebot-agent/src/erniebot_agent/file/file_registry.py @@ -12,14 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -import threading -from typing import Dict, List, Optional +from typing import Dict, List, Optional, final from erniebot_agent.file.base import File -from erniebot_agent.utils.misc import Singleton -class FileRegistry(metaclass=Singleton): +@final +class FileRegistry(object): """ Singleton class for managing file registration. @@ -35,7 +34,6 @@ class FileRegistry(metaclass=Singleton): def __init__(self) -> None: super().__init__() self._id_to_file: Dict[str, File] = {} - self._lock = threading.Lock() def register_file(self, file: File, *, allow_overwrite: bool = False) -> None: """ @@ -53,10 +51,10 @@ def register_file(self, file: File, *, allow_overwrite: bool = False) -> None: """ file_id = file.id - with self._lock: - if not allow_overwrite and file_id in self._id_to_file: - raise RuntimeError(f"ID {repr(file_id)} is already registered.") - self._id_to_file[file_id] = file + if file_id in self._id_to_file: + if not allow_overwrite: + raise ValueError(f"File with ID {repr(file_id)} is already registered.") + self._id_to_file[file_id] = file def unregister_file(self, file: File) -> None: """ @@ -73,10 +71,9 @@ def unregister_file(self, file: File) -> None: """ file_id = file.id - with self._lock: - if file_id not in self._id_to_file: - raise RuntimeError(f"ID {repr(file_id)} is not registered.") - self._id_to_file.pop(file_id) + if file_id not in self._id_to_file: + raise ValueError(f"File with ID {repr(file_id)} is not registered.") + self._id_to_file.pop(file_id) def look_up_file(self, file_id: str) -> Optional[File]: """ @@ -89,8 +86,7 @@ def look_up_file(self, file_id: str) -> Optional[File]: Optional[File]: The File object if found, or None if not found. """ - with self._lock: - return self._id_to_file.get(file_id, None) + return self._id_to_file.get(file_id, None) def list_files(self) -> List[File]: """ @@ -100,19 +96,4 @@ def list_files(self) -> List[File]: List[File]: The list of registered File objects. """ - with self._lock: - return list(self._id_to_file.values()) - - -_file_registry = FileRegistry() - - -def get_file_registry() -> FileRegistry: - """ - Get the global instance of the FileRegistry. - - Returns: - FileRegistry: The global instance of the FileRegistry. - - """ - return _file_registry + return list(self._id_to_file.values()) From b0a52f4ba98ca9a92f092757a025fc3445ac9b63 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 16:21:52 +0800 Subject: [PATCH 42/63] fix typo --- erniebot-agent/src/erniebot_agent/file/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erniebot-agent/src/erniebot_agent/file/README.md b/erniebot-agent/src/erniebot_agent/file/README.md index e07a6c742..c91568f52 100644 --- a/erniebot-agent/src/erniebot_agent/file/README.md +++ b/erniebot-agent/src/erniebot_agent/file/README.md @@ -66,7 +66,7 @@ `RemoteFileClient` 是用于与远程文件服务器交互的类。它定义了文件上传、文件下载、文件删除等操作的方法。`AIStudioFileClient` 是 `RemoteFileClient` 的一个具体推荐实现，用于与文件服务交互，用户使用 `access token`作为参数用于身份验证，之后能够在AIStudio文件服务中上传、检索、列出文件，以及创建临时URL以访问文件。`RemoteFileClient`使用时被 `FileManager`持有，一旦 `FileManager`关闭，`RemoteFileClient`也会相应被关闭，其中的资源也会被相应释放。 -## 6. 使用方法 +## 5. 使用方法 1. 通过 `GlobalFileManagerHandler`获取全局的FileManager，通过它来控制所有文件，注：它的生命周期同整个事件循环。 From c8b65750fae68f1f9a3e711c9cd83af6494b4efd Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 16:26:00 +0800 Subject: [PATCH 43/63] fix typo --- docs/modules/file.md | 8 -------- 1 file changed, 8 deletions(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index 4d76253f8..39031bc0c 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -1,11 +1,3 @@ - # File 模块介绍 ## 1. File 简介 From 13ac7de19a46307e2d511055bdbff3bf1b2f1605 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 16:34:02 +0800 Subject: [PATCH 44/63] add global file manager methods --- .../erniebot_agent/file/global_file_manager_handler.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py b/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py index 9d654c0d6..a05821fba 100644 --- a/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py +++ b/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py @@ -31,9 +31,11 @@ class GlobalFileManagerHandler(Singleton): and allows for its configuration and retrieval. - Attributes: - _file_manager (Optional[FileManager]): The global FileManager instance. - _lock (asyncio.Lock): A lock for ensuring thread-safe access to the FileManager. + Methods: + get: Asynchronously retrieves the global FileManager instance. + configure: Asynchronously configures the global FileManager at beginning + of event loop. + set: Asynchronously sets the global FileManager explicitly. """ From e2132e479ee18caebd2c41b3af2703e2f4104e3d Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 16:35:16 +0800 Subject: [PATCH 45/63] add global file manager methods --- .../src/erniebot_agent/file/global_file_manager_handler.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py b/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py index a05821fba..1af2656c1 100644 --- a/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py +++ b/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py @@ -33,8 +33,8 @@ class GlobalFileManagerHandler(Singleton): Methods: get: Asynchronously retrieves the global FileManager instance. - configure: Asynchronously configures the global FileManager at beginning - of event loop. + configure: Asynchronously configures the global FileManager + at the beginning of event loop. set: Asynchronously sets the global FileManager explicitly. """ @@ -79,7 +79,8 @@ async def configure( """ Configure the global FileManager. - This method configures the global FileManager with the provided parameters. + This method configures the global FileManager with the provided parameters + at the beginning of event loop. If the global FileManager is already set, it raises an error. Args: From 433ab17156d6b86c4ae685ba846e62e2a566a36b Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 16:40:17 +0800 Subject: [PATCH 46/63] small fix --- erniebot-agent/src/erniebot_agent/file/__init__.py | 4 ++++ erniebot-agent/src/erniebot_agent/file/local_file.py | 1 + 2 files changed, 5 insertions(+) diff --git a/erniebot-agent/src/erniebot_agent/file/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py index 53c3eeb94..755d4c5bf 100644 --- a/erniebot-agent/src/erniebot_agent/file/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file/__init__.py @@ -38,6 +38,10 @@ >>> from erniebot_agent.file import GlobalFileManagerHandler >>> async def demo_function(): >>> file_manager = await GlobalFileManagerHandler().get() + >>> local_file = await file_manager.create_file_from_path(file_path='your_path', file_type='local') + + >>> file = file_manager.look_up_file_by_id(file_id='your_file_id') + >>> file_content = await file.read_contents() """ from .global_file_manager_handler import GlobalFileManagerHandler diff --git a/erniebot-agent/src/erniebot_agent/file/local_file.py b/erniebot-agent/src/erniebot_agent/file/local_file.py index 3e80b8784..e58ec8463 100644 --- a/erniebot-agent/src/erniebot_agent/file/local_file.py +++ b/erniebot-agent/src/erniebot_agent/file/local_file.py @@ -125,6 +125,7 @@ def __init__( self.path = path async def read_contents(self) -> bytes: + """Asynchronously read the contents of the local file.""" return await anyio.Path(self.path).read_bytes() def _get_attrs_str(self) -> str: From 7d45c2ff1560148bd9a9ffc3051a038263f5646d Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 17:06:53 +0800 Subject: [PATCH 47/63] fix mypy --- erniebot-agent/src/erniebot_agent/file/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/erniebot-agent/src/erniebot_agent/file/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py index 755d4c5bf..8becef00a 100644 --- a/erniebot-agent/src/erniebot_agent/file/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file/__init__.py @@ -39,7 +39,7 @@ >>> async def demo_function(): >>> file_manager = await GlobalFileManagerHandler().get() >>> local_file = await file_manager.create_file_from_path(file_path='your_path', file_type='local') - + >>> file = file_manager.look_up_file_by_id(file_id='your_file_id') >>> file_content = await file.read_contents() """ From 0d84d29df02c7f3849019ea0fa639ff78dff902d Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 17:13:29 +0800 Subject: [PATCH 48/63] add detailed des of file module --- docs/modules/file.md | 4 ++-- erniebot-agent/src/erniebot_agent/file/README.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index 39031bc0c..f1ab98d36 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -2,7 +2,7 @@ ## 1. File 简介 -文件管理模块提供了用于管理文件的一系列类，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。 +文件管理模块提供了用于管理文件的一系列类方便用户与Agent进行交互，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作，**不推荐**用户自行操作 `File`类以免造成资源泄露，同时从效率方面考虑，`FileManager`将作为此模块中生命周期最长的对象，请不要随意关闭它。 ## 2. File 基类介绍 @@ -75,7 +75,7 @@ from erniebot_agent.file import GlobalFileManagerHandler async def demo_function(): file_manager = await GlobalFileManagerHandler().get() ``` - + 2. 通过 `GlobalFileManagerHandler`创建 `File` ```python diff --git a/erniebot-agent/src/erniebot_agent/file/README.md b/erniebot-agent/src/erniebot_agent/file/README.md index c91568f52..53d5c2301 100644 --- a/erniebot-agent/src/erniebot_agent/file/README.md +++ b/erniebot-agent/src/erniebot_agent/file/README.md @@ -2,7 +2,7 @@ ## 1. File 简介 -文件管理模块提供了用于管理文件的一系列类，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。 +文件管理模块提供了用于管理文件的一系列类方便用户与Agent进行交互，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作，**不推荐**用户自行操作 `File`类以免造成资源泄露，同时从效率方面考虑，`FileManager`将作为此模块中生命周期最长的对象，请不要随意关闭它。 ## 2. File 基类介绍 From cfe922c45a5458b6daa50c1848860186913f736e Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 17:17:01 +0800 Subject: [PATCH 49/63] add detailed des of file module --- docs/modules/file.md | 2 +- erniebot-agent/src/erniebot_agent/file/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index f1ab98d36..67f2fc584 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -2,7 +2,7 @@ ## 1. File 简介 -文件管理模块提供了用于管理文件的一系列类方便用户与Agent进行交互，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作，**不推荐**用户自行操作 `File`类以免造成资源泄露，同时从效率方面考虑，`FileManager`将作为此模块中生命周期最长的对象，请不要随意关闭它。 +文件管理模块提供了用于管理文件的一系列类方便用户与Agent进行交互，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作，**不推荐**用户自行操作 `File`类以免造成资源泄露，同时从效率方面考虑，`FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。 ## 2. File 基类介绍 diff --git a/erniebot-agent/src/erniebot_agent/file/README.md b/erniebot-agent/src/erniebot_agent/file/README.md index 53d5c2301..86756d871 100644 --- a/erniebot-agent/src/erniebot_agent/file/README.md +++ b/erniebot-agent/src/erniebot_agent/file/README.md @@ -2,7 +2,7 @@ ## 1. File 简介 -文件管理模块提供了用于管理文件的一系列类方便用户与Agent进行交互，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作，**不推荐**用户自行操作 `File`类以免造成资源泄露，同时从效率方面考虑，`FileManager`将作为此模块中生命周期最长的对象，请不要随意关闭它。 +文件管理模块提供了用于管理文件的一系列类方便用户与Agent进行交互，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作，**不推荐**用户自行操作 `File`类以免造成资源泄露，同时从效率方面考虑，`FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。 ## 2. File 基类介绍 From acc63c05d9d0a081d76853e1dc54cced2a3e4a79 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 17:17:39 +0800 Subject: [PATCH 50/63] add detailed des of file module --- docs/modules/file.md | 2 +- erniebot-agent/src/erniebot_agent/file/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index 67f2fc584..237b05e5d 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -2,7 +2,7 @@ ## 1. File 简介 -文件管理模块提供了用于管理文件的一系列类方便用户与Agent进行交互，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作，**不推荐**用户自行操作 `File`类以免造成资源泄露，同时从效率方面考虑，`FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。 +文件管理模块提供了用于管理文件的一系列类方便用户与Agent进行交互，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作，**不推荐**用户自行操作 `File`类以免造成资源泄露。同时从效率方面考虑，`FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。 ## 2. File 基类介绍 diff --git a/erniebot-agent/src/erniebot_agent/file/README.md b/erniebot-agent/src/erniebot_agent/file/README.md index 86756d871..6e85a2e58 100644 --- a/erniebot-agent/src/erniebot_agent/file/README.md +++ b/erniebot-agent/src/erniebot_agent/file/README.md @@ -2,7 +2,7 @@ ## 1. File 简介 -文件管理模块提供了用于管理文件的一系列类方便用户与Agent进行交互，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作，**不推荐**用户自行操作 `File`类以免造成资源泄露，同时从效率方面考虑，`FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。 +文件管理模块提供了用于管理文件的一系列类方便用户与Agent进行交互，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作，**不推荐**用户自行操作 `File`类以免造成资源泄露。同时从效率方面考虑，`FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。 ## 2. File 基类介绍 From 3e9f6091caa5560a4eb7151677a6e29d0d52a1b6 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Mon, 25 Dec 2023 20:01:05 +0800 Subject: [PATCH 51/63] rm readme of file --- .../src/erniebot_agent/file/README.md | 95 ------------------- 1 file changed, 95 deletions(-) delete mode 100644 erniebot-agent/src/erniebot_agent/file/README.md diff --git a/erniebot-agent/src/erniebot_agent/file/README.md b/erniebot-agent/src/erniebot_agent/file/README.md deleted file mode 100644 index 6e85a2e58..000000000 --- a/erniebot-agent/src/erniebot_agent/file/README.md +++ /dev/null @@ -1,95 +0,0 @@ -# File 模块介绍 - -## 1. File 简介 - -文件管理模块提供了用于管理文件的一系列类方便用户与Agent进行交互，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作，**不推荐**用户自行操作 `File`类以免造成资源泄露。同时从效率方面考虑，`FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。 - -## 2. File 基类介绍 - -`File` 类是文件管理模块的基础类，用于表示通用的文件对象（不建议自行创建 `File` 类以免无法被 `Agent`识别）。它包含文件的基本属性，如文件ID、文件名、文件大小、创建时间、文件用途和文件元数据。此外， `File `类还定义了一系列抽象方法，比较常用的有：异步读取文件内容的 `read_contents `方法，以及将文件内容写入本地路径的 `write_contents_to `方法以及一些辅助方法：生成文件的字符串表示形式、转换为字典形式等。在File类的内部，其主要有两个继承子类，一个是 `Local File `，一个是 `Remote File `。以下是 `File` 基类的属性以及方法介绍。 - -| 属性 | 类型 | 描述 | -| ---------- | -------------- | --------------------------------------------------------- | -| id | str | 文件的唯一标识符 | -| filename | str | 文件名 | -| byte_size | int | 文件大小（以字节为单位） | -| created_at | str | 文件创建时间的时间戳 | -| purpose | str | 文件的目的或用途，有"assistants", "assistants_output"两种 | -| metadata | Dict[str, Any] | 与文件相关的附加元数据 | - -| 方法 | 描述 | -| ----------------- | ------------------------------ | -| read_contents | 异步读取文件内容 | -| write_contents_to | 异步将文件内容写入本地路径 | -| get_file_repr | 返回用于特定上下文的字符串表示 | -| to_dict | 将File对象转换为字典 | - -### 2.2 File 子类 - -#### 2.2.1 LocalFile 类 - -`LocalFile` 是 `File` 的子类，表示本地文件。除了继承自基类的属性外，它还添加了文件路径属性 `path`，用于表示文件在本地文件系统中的路径。 - -#### 2.2.2 RemoteFile 类 - -`RemoteFile` 也是 `File` 的子类，表示远程文件。它与 `LocalFile` 不同之处在于，它的文件内容存储在远程文件服务器交。`RemoteFile` 类还包含与远程文件服务器交互的相关逻辑。 - -## 3. FileManager 类介绍 - -`FileManager` 类是一个高级文件管理工具，封装了文件的创建、上传、删除等高级操作，以及与 `Agent`进行交互，无论是 `Local File `还是 `RemoteFIle `都可以使用它来统一管理。`FileManager `集成了与远程文件服务器交互的逻辑，通过 `RemoteFileClient `完成上传、下载、删除等文件操作以及与本地文件交互的逻辑，从本地地址创建 `Local File `。它依赖于 `FileRegistry` 来对文件进行用于在整个应用程序中管理文件的注册和查找。 - -以下是相关的属性和方法 - -| 属性 | 类型 | 描述 | -| ------------------ | ------------------ | ------------------------ | -| remote_file_client | RemoteFileClient | 远程文件客户端。 | -| save_dir | Optional[FilePath] | 用于保存本地文件的目录。 | -| closed | bool | 文件管理器是否已关闭。 | - -| 方法 | 描述 | -| ---------------------------- | ------------------------------------ | -| create_file_from_path | 从指定文件路径创建文件 | -| create_local_file_from_path | 从文件路径创建本地文件 | -| create_remote_file_from_path | 从文件路径创建远程文件并上传至服务器 | -| create_file_from_bytes | 从字节创建文件 | -| retrieve_remote_file_by_id | 通过ID获取远程文件 | -| look_up_file_by_id | 通过ID查找本地文件 | -| list_remote_files | 列出远程文件 | - -注： - -* `FileManager` 类不可被复制以免造成资源泄露。 -* 如果未指定 `save_dir`，那么当 `FileManager`关闭时，所有与之关联的本地文件都会被回收。 -* 如果 `FileManager` 类有相关联的 `RemoteFileClient`，那么当 `FileManager`关闭时，相关联的 `RemoteFileClient`也会一起关闭。 - -## 4. RemoteFileClient 类介绍 - -`RemoteFileClient` 是用于与远程文件服务器交互的类。它定义了文件上传、文件下载、文件删除等操作的方法。`AIStudioFileClient` 是 `RemoteFileClient` 的一个具体推荐实现，用于与文件服务交互，用户使用 `access token`作为参数用于身份验证，之后能够在AIStudio文件服务中上传、检索、列出文件，以及创建临时URL以访问文件。`RemoteFileClient`使用时被 `FileManager`持有，一旦 `FileManager`关闭，`RemoteFileClient`也会相应被关闭，其中的资源也会被相应释放。 - -## 5. 使用方法 - -1. 通过 `GlobalFileManagerHandler`获取全局的FileManager，通过它来控制所有文件，注：它的生命周期同整个事件循环。 - - ```python - from erniebot_agent.file import GlobalFileManagerHandler - async def demo_function(): - file_manager = await GlobalFileManagerHandler().get() - ``` -2. 通过 `GlobalFileManagerHandler`创建 `File` - - ```python - from erniebot_agent.file import GlobalFileManagerHandler - async def demo_function(): - file_manager = await GlobalFileManagerHandler().get() - # 从路径创建File, file_type可选择local或者remote file_purpose='assistant'代表用于给LLM输入使用 - local_file = await file_manager.create_file_from_path(file_path='your_path', file_type='local') - ``` -3. 通过 `GlobalFileManagerHandler`搜索 `File` - - ```python - from erniebot_agent.file import GlobalFileManagerHandler - async def demo_function(): - file_manager = await GlobalFileManagerHandler().get() - file = file_manager.look_up_file_by_id(file_id='your_file_id') - file_content = await file.read_contents() - ``` From ab29585cff5bc220b564d807a7c59c85c7a08781 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Tue, 26 Dec 2023 09:38:43 +0800 Subject: [PATCH 52/63] fix mkdocs long line --- docs/package/erniebot_agent/file.md | 20 +++++++++++++++++++ .../src/erniebot_agent/file/file_manager.py | 2 +- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/docs/package/erniebot_agent/file.md b/docs/package/erniebot_agent/file.md index d8fe7c467..19a087a19 100644 --- a/docs/package/erniebot_agent/file.md +++ b/docs/package/erniebot_agent/file.md @@ -3,11 +3,16 @@ ::: erniebot_agent.file options: summary: true + members: + - ::: erniebot_agent.file.base options: summary: true + separate_signature: true + show_signature_annotations: true + line_length: 60 members: - File @@ -15,29 +20,44 @@ ::: erniebot_agent.file.local_file options: summary: true + separate_signature: true + show_signature_annotations: true + line_length: 60 members: - LocalFile ::: erniebot_agent.file.remote_file options: summary: true + separate_signature: true + show_signature_annotations: true + line_length: 60 members: - RemoteFile ::: erniebot_agent.file.file_manager options: summary: true + separate_signature: true + show_signature_annotations: true + line_length: 60 members: - FileManager ::: erniebot_agent.file.global_file_manager_handler options: summary: true + separate_signature: true + show_signature_annotations: true + line_length: 60 members: - GlobalFileManagerHandler ::: erniebot_agent.file.remote_file options: summary: true + separate_signature: true + show_signature_annotations: true + line_length: 60 members: - AIStudioFileClient \ No newline at end of file diff --git a/erniebot-agent/src/erniebot_agent/file/file_manager.py b/erniebot-agent/src/erniebot_agent/file/file_manager.py index 0da6eba85..196875d46 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/file_manager.py @@ -366,7 +366,7 @@ def look_up_file_by_id(self, file_id: str) -> File: file_id (str): The ID of the file. Returns: - file[File]: The looked-up file, or None if not found. + file[File]: The looked-up file. Raises: FileError: If the file with the specified ID is not found. From d2b280443a367c5bbaa293120354300be97824c3 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Tue, 26 Dec 2023 17:25:25 +0800 Subject: [PATCH 53/63] renew file.md des --- docs/modules/file.md | 42 +++++++++++++++---- .../src/erniebot_agent/file/__init__.py | 2 +- .../src/erniebot_agent/file/base.py | 4 +- .../src/erniebot_agent/file/file_manager.py | 3 +- 4 files changed, 40 insertions(+), 11 deletions(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index 237b05e5d..5054029ea 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -1,12 +1,32 @@ -# File 模块介绍 +# File 模块 -## 1. File 简介 +## 1. File 模块简介 -文件管理模块提供了用于管理文件的一系列类方便用户与Agent进行交互，其中包括 `File` 及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作，**不推荐**用户自行操作 `File`类以免造成资源泄露。同时从效率方面考虑，`FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。 +文件管理模块提供了用于管理文件的一系列类，方便用户与Agent进行交互，其中包括 `File` 基类及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作以及获取Agent产生的文件。 -## 2. File 基类介绍 +注： + +* **不推荐**用户自行操作 `File`类以免造成资源泄露。 +* `FileManager`操作文件主要用于异步函数中，在同步函数中使用可能会无效。 +* `FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。如果需要关闭已停止对其中所有注册文件的使用。 + +## 2. File 基类及其子类介绍 + +`File` 类是文件管理模块的基础类，用于表示通用的文件对象（不建议自行创建 `File` 类以免无法被 `Agent`识别使用以及无法被回收）。它包含文件的基本属性，如文件ID、文件名、文件大小、创建时间、文件用途和文件元数据。 + +此外， `File`类还定义了一系列抽象方法，比较常用的有： + +* 异步读取文件内容的 `read_contents`方法 +* 将文件内容写入本地路径的 `write_contents_to`方法 -`File` 类是文件管理模块的基础类，用于表示通用的文件对象（不建议自行创建 `File` 类以免无法被 `Agent`识别）。它包含文件的基本属性，如文件ID、文件名、文件大小、创建时间、文件用途和文件元数据。此外， `File `类还定义了一系列抽象方法，比较常用的有：异步读取文件内容的 `read_contents `方法，以及将文件内容写入本地路径的 `write_contents_to `方法以及一些辅助方法：生成文件的字符串表示形式、转换为字典形式等。在File类的内部，其主要有两个继承子类，一个是 `Local File `，一个是 `Remote File `。以下是 `File` 基类的属性以及方法介绍。 +以及一些辅助方法： + +* 生成文件的字符串表示形式 +* 转换为字典形式 + +在File类的内部，其主要有两个继承子类，一个是 `Local File`，一个是 `Remote File`。 + +以下是 `File` 基类的属性以及方法介绍： | 属性 | 类型 | 描述 | | ---------- | -------------- | --------------------------------------------------------- | @@ -36,7 +56,7 @@ ## 3. FileManager 类介绍 -`FileManager` 类是一个高级文件管理工具，封装了文件的创建、上传、删除等高级操作，以及与 `Agent`进行交互，无论是 `Local File `还是 `RemoteFIle `都可以使用它来统一管理。`FileManager `集成了与远程文件服务器交互的逻辑，通过 `RemoteFileClient `完成上传、下载、删除等文件操作以及与本地文件交互的逻辑，从本地地址创建 `Local File `。它依赖于 `FileRegistry` 来对文件进行用于在整个应用程序中管理文件的注册和查找。 +`FileManager` 类是一个高级文件管理工具，封装了文件的创建、上传、删除等操作，以及与 `Agent`进行交互，无论是 `LocalFile`还是 `RemoteFile`都可以使用它来统一管理。`FileManage`集成了与远程文件服务器交互的逻辑（通过 `RemoteFileClient`完成上传、下载、删除等文件操作）以及与本地文件交互的逻辑（从本地路径创建 `LocalFile`）。它依赖于 `FileRegistry` 来对文件进行用于在整个应用程序中管理文件的注册和查找。 以下是相关的属性和方法 @@ -66,6 +86,10 @@ `RemoteFileClient` 是用于与远程文件服务器交互的类。它定义了文件上传、文件下载、文件删除等操作的方法。`AIStudioFileClient` 是 `RemoteFileClient` 的一个具体推荐实现，用于与文件服务交互，用户使用 `access token`作为参数用于身份验证，之后能够在AIStudio文件服务中上传、检索、列出文件，以及创建临时URL以访问文件。`RemoteFileClient`使用时被 `FileManager`持有，一旦 `FileManager`关闭，`RemoteFileClient`也会相应被关闭，其中的资源也会被相应释放。 +注： + +* 一般情况下无需使用 `RemoteFile`，默认所有文件都为 `LocalFile`，如需使用，将该对象传入 `FileManager`即可。 + ## 5. 使用方法 1. 通过 `GlobalFileManagerHandler`获取全局的FileManager，通过它来控制所有文件，注：它的生命周期同整个事件循环。 @@ -86,12 +110,16 @@ async def demo_function(): local_file = await file_manager.create_file_from_path(file_path='your_path', file_type='local') ``` -3. 通过 `GlobalFileManagerHandler`搜索 `File` +3. 通过 `GlobalFileManagerHandler`搜索以及保存 `File` ```python from erniebot_agent.file import GlobalFileManagerHandler async def demo_function(): file_manager = await GlobalFileManagerHandler().get() + # 通过fileid搜索文件 file = file_manager.look_up_file_by_id(file_id='your_file_id') + # 读取file内容(bytes) file_content = await file.read_contents() + # 写出到指定位置 + await local_file.write_contents_to('your_willing_path') ``` diff --git a/erniebot-agent/src/erniebot_agent/file/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py index 8becef00a..3a276ba14 100644 --- a/erniebot-agent/src/erniebot_agent/file/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file/__init__.py @@ -17,7 +17,7 @@ We use it to manage `File` obeject, including `LocalFile` and `RemoteFile`. -A few notes about the current state of this submodule: +A few notes about this submodule: - If you do not set environment variable `AISTUDIO_ACCESS_TOKEN`, it will be under default setting. diff --git a/erniebot-agent/src/erniebot_agent/file/base.py b/erniebot-agent/src/erniebot_agent/file/base.py index 53b6db88d..e641585e5 100644 --- a/erniebot-agent/src/erniebot_agent/file/base.py +++ b/erniebot-agent/src/erniebot_agent/file/base.py @@ -28,7 +28,7 @@ class File(metaclass=abc.ABCMeta): filename (str): File name. byte_size (int): Size of the file in bytes. created_at (str): Timestamp indicating the file creation time. - purpose (str): Purpose or use case of the file. [] + purpose (str): Purpose or use case of the file. metadata (Dict[str, Any]): Additional metadata associated with the file. Methods: @@ -56,7 +56,7 @@ def __init__( filename (str): File name. byte_size (int): Size of the file in bytes. created_at (str): Timestamp indicating the file creation time. - purpose (str): Purpose or use case of the file. [] + purpose (str): Purpose or use case of the file. metadata (Dict[str, Any]): Additional metadata associated with the file. Returns: diff --git a/erniebot-agent/src/erniebot_agent/file/file_manager.py b/erniebot-agent/src/erniebot_agent/file/file_manager.py index 196875d46..45f32ea54 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/file_manager.py @@ -83,7 +83,8 @@ def __init__( Args: remote_file_client (Optional[RemoteFileClient]): The remote file client. - auto_register (bool): Automatically register files in the file registry. + prune_on_close (bool): Control whether to automatically clean up files + that can be safely deleted when the object is closed. save_dir (Optional[FilePath]): Directory for saving local files. Returns: From 709c11b9fd08c320ecad071a18df508363130a2d Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Tue, 26 Dec 2023 18:19:48 +0800 Subject: [PATCH 54/63] add examples --- docs/modules/file.md | 27 +++++++++++++------ .../src/erniebot_agent/file/__init__.py | 16 ++++++++--- .../src/erniebot_agent/file/base.py | 4 +-- .../src/erniebot_agent/file/file_manager.py | 2 +- .../file/global_file_manager_handler.py | 6 ++--- 5 files changed, 38 insertions(+), 17 deletions(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index 5054029ea..66bb79381 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -2,9 +2,9 @@ ## 1. File 模块简介 -文件管理模块提供了用于管理文件的一系列类，方便用户与Agent进行交互，其中包括 `File` 基类及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作以及获取Agent产生的文件。 +文件管理模块提供了用于管理文件的一系列类，方便用户与Agent进行交互，其中包括 `File` 基类及其子类、`FileManager` 、`GlobalFileManagerHandler`以及与远程文件服务器交互的 `RemoteFileClient`。 -注： +推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作以及获取Agent产生的文件。 * **不推荐**用户自行操作 `File`类以免造成资源泄露。 * `FileManager`操作文件主要用于异步函数中，在同步函数中使用可能会无效。 @@ -76,19 +76,15 @@ | look_up_file_by_id | 通过ID查找本地文件 | | list_remote_files | 列出远程文件 | -注： - * `FileManager` 类不可被复制以免造成资源泄露。 -* 如果未指定 `save_dir`，那么当 `FileManager`关闭时，所有与之关联的本地文件都会被回收。 +* 如果未指定 `save_dir`，那么当 `FileManager`关闭时，所有与之关联的本地文件都会被回收。反之，都会被保存。 * 如果 `FileManager` 类有相关联的 `RemoteFileClient`，那么当 `FileManager`关闭时，相关联的 `RemoteFileClient`也会一起关闭。 ## 4. RemoteFileClient 类介绍 `RemoteFileClient` 是用于与远程文件服务器交互的类。它定义了文件上传、文件下载、文件删除等操作的方法。`AIStudioFileClient` 是 `RemoteFileClient` 的一个具体推荐实现，用于与文件服务交互，用户使用 `access token`作为参数用于身份验证，之后能够在AIStudio文件服务中上传、检索、列出文件，以及创建临时URL以访问文件。`RemoteFileClient`使用时被 `FileManager`持有，一旦 `FileManager`关闭，`RemoteFileClient`也会相应被关闭，其中的资源也会被相应释放。 -注： - -* 一般情况下无需使用 `RemoteFile`，默认所有文件都为 `LocalFile`，如需使用，将该对象传入 `FileManager`即可。 +* 一般情况下无需使用 `RemoteFile`，默认所有文件都为 `LocalFile`，如需使用，将 `GlobalFileManagerHandler`的。 ## 5. 使用方法 @@ -96,6 +92,7 @@ ```python from erniebot_agent.file import GlobalFileManagerHandler + async def demo_function(): file_manager = await GlobalFileManagerHandler().get() ``` @@ -104,6 +101,7 @@ async def demo_function(): ```python from erniebot_agent.file import GlobalFileManagerHandler + async def demo_function(): file_manager = await GlobalFileManagerHandler().get() # 从路径创建File, file_type可选择local或者remote file_purpose='assistant'代表用于给LLM输入使用 @@ -114,6 +112,7 @@ async def demo_function(): ```python from erniebot_agent.file import GlobalFileManagerHandler + async def demo_function(): file_manager = await GlobalFileManagerHandler().get() # 通过fileid搜索文件 @@ -123,3 +122,15 @@ async def demo_function(): # 写出到指定位置 await local_file.write_contents_to('your_willing_path') ``` + +4. 配置 `GlobalFileManagerHandler`从而在Agent中直接获取相关文件 + ```python + from erniebot_agent.file import GlobalFileManagerHandler + + async def demo_function(): + await GlobalFileManagerHandler().configure(save_dir='your_path') # 需要在事件循环最开始配置 + ... # 此处省略agent创建过程 + response = await agent.async_run('请帮我画一张北京市的图') + # 您可以通过AgentResponse.files获取agent所有文件也可以在save_dir中找到生成的图片 + files = response.files + ``` diff --git a/erniebot-agent/src/erniebot_agent/file/__init__.py b/erniebot-agent/src/erniebot_agent/file/__init__.py index 3a276ba14..46fd483ae 100644 --- a/erniebot-agent/src/erniebot_agent/file/__init__.py +++ b/erniebot-agent/src/erniebot_agent/file/__init__.py @@ -13,9 +13,16 @@ # limitations under the License. """ -File module is used to manage the file system by a global file manager: `GlobalFileManagerHandler`. +The file module provides a series of classes for managing files, +which facilitate user interaction with the agent, +including the File base class and its subclasses, FileManager, GlobalFileManagerHandler, +and RemoteFileClient, which interacts with remote file servers. + +It is recommended to use the GlobalFileManagerHandler to initialize the FileManager +and obtain the global FileManager at the beginning of the event loop. +Afterwards, you can simply use this global FileManager to perform operations such as adding, +deleting, and querying files, as well as obtaining files generated by the agent. -We use it to manage `File` obeject, including `LocalFile` and `RemoteFile`. A few notes about this submodule: @@ -37,11 +44,14 @@ Examples: >>> from erniebot_agent.file import GlobalFileManagerHandler >>> async def demo_function(): + >>> # need to use at the beginning of event loop + >>> await GlobalFileManagerHandler().configure(save_dir='your_path') >>> file_manager = await GlobalFileManagerHandler().get() >>> local_file = await file_manager.create_file_from_path(file_path='your_path', file_type='local') >>> file = file_manager.look_up_file_by_id(file_id='your_file_id') - >>> file_content = await file.read_contents() + >>> file_content = await file.read_contents() # get file content(bytes) + >>> await local_file.write_contents_to('your_willing_path') # save to location you want """ from .global_file_manager_handler import GlobalFileManagerHandler diff --git a/erniebot-agent/src/erniebot_agent/file/base.py b/erniebot-agent/src/erniebot_agent/file/base.py index e641585e5..4ccc002cf 100644 --- a/erniebot-agent/src/erniebot_agent/file/base.py +++ b/erniebot-agent/src/erniebot_agent/file/base.py @@ -28,7 +28,7 @@ class File(metaclass=abc.ABCMeta): filename (str): File name. byte_size (int): Size of the file in bytes. created_at (str): Timestamp indicating the file creation time. - purpose (str): Purpose or use case of the file. + purpose (str): Purpose or use case of the file. metadata (Dict[str, Any]): Additional metadata associated with the file. Methods: @@ -56,7 +56,7 @@ def __init__( filename (str): File name. byte_size (int): Size of the file in bytes. created_at (str): Timestamp indicating the file creation time. - purpose (str): Purpose or use case of the file. + purpose (str): Purpose or use case of the file. metadata (Dict[str, Any]): Additional metadata associated with the file. Returns: diff --git a/erniebot-agent/src/erniebot_agent/file/file_manager.py b/erniebot-agent/src/erniebot_agent/file/file_manager.py index 45f32ea54..2af4db6bd 100644 --- a/erniebot-agent/src/erniebot_agent/file/file_manager.py +++ b/erniebot-agent/src/erniebot_agent/file/file_manager.py @@ -83,7 +83,7 @@ def __init__( Args: remote_file_client (Optional[RemoteFileClient]): The remote file client. - prune_on_close (bool): Control whether to automatically clean up files + prune_on_close (bool): Control whether to automatically clean up files that can be safely deleted when the object is closed. save_dir (Optional[FilePath]): Directory for saving local files. diff --git a/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py b/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py index 524804c6f..b0c3f4574 100644 --- a/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py +++ b/erniebot-agent/src/erniebot_agent/file/global_file_manager_handler.py @@ -38,7 +38,7 @@ class GlobalFileManagerHandler(metaclass=SingletonMeta): set: Asynchronously sets the global FileManager explicitly. """ - + _file_manager: Optional[FileManager] def __init__(self) -> None: @@ -84,9 +84,9 @@ async def configure( If the global FileManager is already set, it raises an error. Args: - access_token (Optional[str]): The access token for remote file management. + access_token (Optional[str]): The access token for remote file client. save_dir (Optional[str]): The directory for saving local files. - enable_remote_file (bool): Whether to enable remote file management. + enable_remote_file (bool): Whether to enable remote file. **opts (Any): Additional options for FileManager. Returns: From 01f4f21362c34aa3fe15f18a73c5d6ea081b9355 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Tue, 26 Dec 2023 18:48:17 +0800 Subject: [PATCH 55/63] fix title typo --- docs/modules/file.md | 6 +++--- docs/package/erniebot_agent/file.md | 1 - 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index 66bb79381..2cbb4e6a5 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -44,13 +44,13 @@ | get_file_repr | 返回用于特定上下文的字符串表示 | | to_dict | 将File对象转换为字典 | -### 2.2 File 子类 +### 2.1 File 子类 -#### 2.2.1 LocalFile 类 +#### 2.1.1 LocalFile 类 `LocalFile` 是 `File` 的子类，表示本地文件。除了继承自基类的属性外，它还添加了文件路径属性 `path`，用于表示文件在本地文件系统中的路径。 -#### 2.2.2 RemoteFile 类 +#### 2.1.2 RemoteFile 类 `RemoteFile` 也是 `File` 的子类，表示远程文件。它与 `LocalFile` 不同之处在于，它的文件内容存储在远程文件服务器交。`RemoteFile` 类还包含与远程文件服务器交互的相关逻辑。 diff --git a/docs/package/erniebot_agent/file.md b/docs/package/erniebot_agent/file.md index 19a087a19..c1fe9d2eb 100644 --- a/docs/package/erniebot_agent/file.md +++ b/docs/package/erniebot_agent/file.md @@ -12,7 +12,6 @@ summary: true separate_signature: true show_signature_annotations: true - line_length: 60 members: - File From 2a414f15dd5916c0675fffc775d5ce21f258e9e8 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Tue, 26 Dec 2023 18:54:32 +0800 Subject: [PATCH 56/63] add mkdocs notes --- docs/modules/file.md | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index 2cbb4e6a5..4e25b977c 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -6,9 +6,10 @@ 推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作以及获取Agent产生的文件。 -* **不推荐**用户自行操作 `File`类以免造成资源泄露。 -* `FileManager`操作文件主要用于异步函数中，在同步函数中使用可能会无效。 -* `FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。如果需要关闭已停止对其中所有注册文件的使用。 +!!! notes 注意 + * **不推荐**用户自行操作 `File`类以免造成资源泄露。 + * `FileManager`操作文件主要用于异步函数中，在同步函数中使用可能会无效。 + * `FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。如果需要关闭已停止对其中所有注册文件的使用。 ## 2. File 基类及其子类介绍 @@ -76,29 +77,28 @@ | look_up_file_by_id | 通过ID查找本地文件 | | list_remote_files | 列出远程文件 | -* `FileManager` 类不可被复制以免造成资源泄露。 -* 如果未指定 `save_dir`，那么当 `FileManager`关闭时，所有与之关联的本地文件都会被回收。反之，都会被保存。 -* 如果 `FileManager` 类有相关联的 `RemoteFileClient`，那么当 `FileManager`关闭时，相关联的 `RemoteFileClient`也会一起关闭。 +!!! notes 注意 + * `FileManager` 类不可被复制以免造成资源泄露。 + * 如果未指定 `save_dir`，那么当 `FileManager`关闭时，所有与之关联的本地文件都会被回收。反之，都会被保存。 + * 如果 `FileManager` 类有相关联的 `RemoteFileClient`，那么当 `FileManager`关闭时，相关联的 `RemoteFileClient`也会一起关闭。 ## 4. RemoteFileClient 类介绍 `RemoteFileClient` 是用于与远程文件服务器交互的类。它定义了文件上传、文件下载、文件删除等操作的方法。`AIStudioFileClient` 是 `RemoteFileClient` 的一个具体推荐实现，用于与文件服务交互，用户使用 `access token`作为参数用于身份验证，之后能够在AIStudio文件服务中上传、检索、列出文件，以及创建临时URL以访问文件。`RemoteFileClient`使用时被 `FileManager`持有，一旦 `FileManager`关闭，`RemoteFileClient`也会相应被关闭，其中的资源也会被相应释放。 -* 一般情况下无需使用 `RemoteFile`，默认所有文件都为 `LocalFile`，如需使用，将 `GlobalFileManagerHandler`的。 +!!! notes 注意 + * 一般情况下无需使用 `RemoteFile`，默认所有文件都为 `LocalFile`，如需使用，将 `GlobalFileManagerHandler`的。 ## 5. 使用方法 1. 通过 `GlobalFileManagerHandler`获取全局的FileManager，通过它来控制所有文件，注：它的生命周期同整个事件循环。 - ```python from erniebot_agent.file import GlobalFileManagerHandler async def demo_function(): file_manager = await GlobalFileManagerHandler().get() ``` - 2. 通过 `GlobalFileManagerHandler`创建 `File` - ```python from erniebot_agent.file import GlobalFileManagerHandler @@ -107,9 +107,7 @@ async def demo_function(): # 从路径创建File, file_type可选择local或者remote file_purpose='assistant'代表用于给LLM输入使用 local_file = await file_manager.create_file_from_path(file_path='your_path', file_type='local') ``` - 3. 通过 `GlobalFileManagerHandler`搜索以及保存 `File` - ```python from erniebot_agent.file import GlobalFileManagerHandler @@ -122,7 +120,6 @@ async def demo_function(): # 写出到指定位置 await local_file.write_contents_to('your_willing_path') ``` - 4. 配置 `GlobalFileManagerHandler`从而在Agent中直接获取相关文件 ```python from erniebot_agent.file import GlobalFileManagerHandler @@ -133,4 +130,4 @@ async def demo_function(): response = await agent.async_run('请帮我画一张北京市的图') # 您可以通过AgentResponse.files获取agent所有文件也可以在save_dir中找到生成的图片 files = response.files - ``` + ``` \ No newline at end of file From 0c217eef2ac31b00bca3158c9b8a3afeac257196 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Tue, 26 Dec 2023 19:12:10 +0800 Subject: [PATCH 57/63] add mkdocs notes --- docs/modules/file.md | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index 4e25b977c..804eb8908 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -7,9 +7,11 @@ 推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作以及获取Agent产生的文件。 !!! notes 注意 - * **不推荐**用户自行操作 `File`类以免造成资源泄露。 - * `FileManager`操作文件主要用于异步函数中，在同步函数中使用可能会无效。 - * `FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。如果需要关闭已停止对其中所有注册文件的使用。 + - **不推荐**用户自行操作 `File`类以免造成资源泄露。 + + - `FileManager`操作文件主要用于异步函数中，在同步函数中使用可能会无效。 + + - `FileManager`将作为此模块中生命周期最长的对象，它会在关闭时回收所有的持有对象（RemoteClient/temp local file），请不要随意关闭它。如果需要关闭已停止对其中所有注册文件的使用。 ## 2. File 基类及其子类介绍 @@ -78,9 +80,11 @@ | list_remote_files | 列出远程文件 | !!! notes 注意 - * `FileManager` 类不可被复制以免造成资源泄露。 - * 如果未指定 `save_dir`，那么当 `FileManager`关闭时，所有与之关联的本地文件都会被回收。反之，都会被保存。 - * 如果 `FileManager` 类有相关联的 `RemoteFileClient`，那么当 `FileManager`关闭时，相关联的 `RemoteFileClient`也会一起关闭。 + - `FileManager` 类不可被复制以免造成资源泄露。 + + - 如果未指定 `save_dir`，那么当 `FileManager`关闭时，所有与之关联的本地文件都会被回收。反之，都会被保存。 + + - 如果 `FileManager` 类有相关联的 `RemoteFileClient`，那么当 `FileManager`关闭时，相关联的 `RemoteFileClient`也会一起关闭。 ## 4. RemoteFileClient 类介绍 @@ -92,13 +96,16 @@ ## 5. 使用方法 1. 通过 `GlobalFileManagerHandler`获取全局的FileManager，通过它来控制所有文件，注：它的生命周期同整个事件循环。 + ```python from erniebot_agent.file import GlobalFileManagerHandler async def demo_function(): file_manager = await GlobalFileManagerHandler().get() ``` + 2. 通过 `GlobalFileManagerHandler`创建 `File` + ```python from erniebot_agent.file import GlobalFileManagerHandler @@ -107,7 +114,9 @@ async def demo_function(): # 从路径创建File, file_type可选择local或者remote file_purpose='assistant'代表用于给LLM输入使用 local_file = await file_manager.create_file_from_path(file_path='your_path', file_type='local') ``` + 3. 通过 `GlobalFileManagerHandler`搜索以及保存 `File` + ```python from erniebot_agent.file import GlobalFileManagerHandler @@ -120,6 +129,7 @@ async def demo_function(): # 写出到指定位置 await local_file.write_contents_to('your_willing_path') ``` + 4. 配置 `GlobalFileManagerHandler`从而在Agent中直接获取相关文件 ```python from erniebot_agent.file import GlobalFileManagerHandler @@ -130,4 +140,4 @@ async def demo_function(): response = await agent.async_run('请帮我画一张北京市的图') # 您可以通过AgentResponse.files获取agent所有文件也可以在save_dir中找到生成的图片 files = response.files - ``` \ No newline at end of file + ``` From 12652bb253c36d6082ca884266cec291e5473d95 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Tue, 26 Dec 2023 19:14:29 +0800 Subject: [PATCH 58/63] add mkdocs notes --- docs/modules/file.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index 804eb8908..c137ac4d4 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -83,7 +83,7 @@ - `FileManager` 类不可被复制以免造成资源泄露。 - 如果未指定 `save_dir`，那么当 `FileManager`关闭时，所有与之关联的本地文件都会被回收。反之，都会被保存。 - + - 如果 `FileManager` 类有相关联的 `RemoteFileClient`，那么当 `FileManager`关闭时，相关联的 `RemoteFileClient`也会一起关闭。 ## 4. RemoteFileClient 类介绍 @@ -141,3 +141,6 @@ async def demo_function(): # 您可以通过AgentResponse.files获取agent所有文件也可以在save_dir中找到生成的图片 files = response.files ``` + +## 6 File的API接口 +`File`模块的API接口，请参考[文档](../../package/erniebot_agent/file/)。 \ No newline at end of file From 6d110efe6f13bcecc290380e5bc5217e0c663fe1 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Tue, 26 Dec 2023 19:15:46 +0800 Subject: [PATCH 59/63] renew style --- docs/modules/file.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/modules/file.md b/docs/modules/file.md index c137ac4d4..a646cb3c8 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -7,6 +7,7 @@ 推荐使用 `GlobalFileManagerHandler`在事件循环开始时初始化 `FileManager`以及获取全局的 `FileManager`，之后只需通过这个全局的 `FileManager`对文件进行增、删、查等操作以及获取Agent产生的文件。 !!! notes 注意 + - **不推荐**用户自行操作 `File`类以免造成资源泄露。 - `FileManager`操作文件主要用于异步函数中，在同步函数中使用可能会无效。 @@ -80,6 +81,7 @@ | list_remote_files | 列出远程文件 | !!! notes 注意 + - `FileManager` 类不可被复制以免造成资源泄露。 - 如果未指定 `save_dir`，那么当 `FileManager`关闭时，所有与之关联的本地文件都会被回收。反之，都会被保存。 From 832ef2235100d6279f660c2769e7f1d139041192 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Tue, 26 Dec 2023 19:18:00 +0800 Subject: [PATCH 60/63] fix typo --- docs/modules/file.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index a646cb3c8..e5663d217 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -60,7 +60,7 @@ ## 3. FileManager 类介绍 -`FileManager` 类是一个高级文件管理工具，封装了文件的创建、上传、删除等操作，以及与 `Agent`进行交互，无论是 `LocalFile`还是 `RemoteFile`都可以使用它来统一管理。`FileManage`集成了与远程文件服务器交互的逻辑（通过 `RemoteFileClient`完成上传、下载、删除等文件操作）以及与本地文件交互的逻辑（从本地路径创建 `LocalFile`）。它依赖于 `FileRegistry` 来对文件进行用于在整个应用程序中管理文件的注册和查找。 +`FileManager` 类是一个高级文件管理工具，封装了文件的创建、上传、删除等操作，以及与 `Agent`进行交互，无论是 `LocalFile`还是 `RemoteFile`都可以使用它来统一管理。`FileManager`集成了与远程文件服务器交互的逻辑（通过 `RemoteFileClient`完成上传、下载、删除等文件操作）以及与本地文件交互的逻辑（从本地路径创建 `LocalFile`）。它依赖于 `FileRegistry` 来对文件进行用于在整个应用程序中管理文件的注册和查找。 以下是相关的属性和方法 From 6e3632b321122a929528f67e98ab72c2bb618f88 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Tue, 26 Dec 2023 19:27:19 +0800 Subject: [PATCH 61/63] fix typo --- docs/modules/file.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index e5663d217..47b13a71d 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -93,7 +93,7 @@ `RemoteFileClient` 是用于与远程文件服务器交互的类。它定义了文件上传、文件下载、文件删除等操作的方法。`AIStudioFileClient` 是 `RemoteFileClient` 的一个具体推荐实现，用于与文件服务交互，用户使用 `access token`作为参数用于身份验证，之后能够在AIStudio文件服务中上传、检索、列出文件，以及创建临时URL以访问文件。`RemoteFileClient`使用时被 `FileManager`持有，一旦 `FileManager`关闭，`RemoteFileClient`也会相应被关闭，其中的资源也会被相应释放。 !!! notes 注意 - * 一般情况下无需使用 `RemoteFile`，默认所有文件都为 `LocalFile`，如需使用，将 `GlobalFileManagerHandler`的。 + * 一般情况下无需使用 `RemoteFile`，默认所有文件都为 `LocalFile`，如需使用，将 `GlobalFileManagerHandler`的`enable_remote_file`设置为True即可。 ## 5. 使用方法 From 47cbb80d72dd5cff87bbfd10c353a7fba66465f1 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Tue, 26 Dec 2023 19:28:02 +0800 Subject: [PATCH 62/63] fix typo --- docs/modules/file.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index 47b13a71d..ee5a22229 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -66,9 +66,9 @@ | 属性 | 类型 | 描述 | | ------------------ | ------------------ | ------------------------ | -| remote_file_client | RemoteFileClient | 远程文件客户端。 | -| save_dir | Optional[FilePath] | 用于保存本地文件的目录。 | -| closed | bool | 文件管理器是否已关闭。 | +| remote_file_client | RemoteFileClient | 远程文件客户端 | +| save_dir | Optional[FilePath] | 用于保存本地文件的目录 | +| closed | bool | 文件管理器是否已关闭 | | 方法 | 描述 | | ---------------------------- | ------------------------------------ | From cee6946fcda771bb0c06afeea6fdc46f67702208 Mon Sep 17 00:00:00 2001 From: Southpika <513923576@qq.com> Date: Tue, 26 Dec 2023 19:28:58 +0800 Subject: [PATCH 63/63] fix typo --- docs/modules/file.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/docs/modules/file.md b/docs/modules/file.md index ee5a22229..255e3c6d7 100644 --- a/docs/modules/file.md +++ b/docs/modules/file.md @@ -105,7 +105,6 @@ from erniebot_agent.file import GlobalFileManagerHandler async def demo_function(): file_manager = await GlobalFileManagerHandler().get() ``` - 2. 通过 `GlobalFileManagerHandler`创建 `File` ```python @@ -116,7 +115,6 @@ async def demo_function(): # 从路径创建File, file_type可选择local或者remote file_purpose='assistant'代表用于给LLM输入使用 local_file = await file_manager.create_file_from_path(file_path='your_path', file_type='local') ``` - 3. 通过 `GlobalFileManagerHandler`搜索以及保存 `File` ```python @@ -131,7 +129,6 @@ async def demo_function(): # 写出到指定位置 await local_file.write_contents_to('your_willing_path') ``` - 4. 配置 `GlobalFileManagerHandler`从而在Agent中直接获取相关文件 ```python from erniebot_agent.file import GlobalFileManagerHandler