diff --git a/.env.example b/.env.example index 254938a..dddad9c 100644 --- a/.env.example +++ b/.env.example @@ -1,5 +1,3 @@ -DENDRITE_API_KEY= -OPENAI_API_KEY= ANTHROPIC_API_KEY= # If using Browserbase diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 226dba5..62a6457 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -49,7 +49,6 @@ jobs: test: runs-on: ubuntu-latest env: - DENDRITE_API_KEY: ${{secrets.DENDRITE_API_KEY}} BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }} BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }} steps: diff --git a/dendrite/browser/async_api/dendrite_browser.py b/dendrite/browser/async_api/dendrite_browser.py index ca51778..fb2db0c 100644 --- a/dendrite/browser/async_api/dendrite_browser.py +++ b/dendrite/browser/async_api/dendrite_browser.py @@ -61,23 +61,16 @@ class AsyncDendrite( AsyncDendrite is a class that manages a browser instance using Playwright, allowing interactions with web pages using natural language. - This class handles initialization with API keys for Dendrite, OpenAI, and Anthropic, manages browser - contexts, and provides methods for navigation, authentication, and other browser-related tasks. + This class handles initialization with configuration options, manages browser contexts, + and provides methods for navigation, authentication, and other browser-related tasks. Attributes: - id (UUID): The unique identifier for the AsyncDendrite instance. - auth_data (Optional[AuthSession]): The authentication session data for the browser. - dendrite_api_key (str): The API key for Dendrite, used for interactions with the Dendrite API. - playwright_options (dict): Options for configuring the Playwright browser instance. - playwright (Optional[Playwright]): The Playwright instance managing the browser. + id (str): The unique identifier for the AsyncDendrite instance. browser_context (Optional[BrowserContext]): The current browser context, which may include cookies and other session data. active_page_manager (Optional[PageManager]): The manager responsible for handling active pages within the browser context. user_id (Optional[str]): The user ID associated with the browser session. - browser_api_client (BrowserAPIClient): The API client used for communicating with the Dendrite API. - api_config (APIConfig): The configuration for the language models, including API keys for OpenAI and Anthropic. - - Raises: - Exception: If any of the required API keys (Dendrite, OpenAI, Anthropic) are not provided or found in the environment variables. + logic_engine (AsyncLogicEngine): The engine used for processing natural language interactions. + closed (bool): Whether the browser instance has been closed. """ def __init__( @@ -94,10 +87,14 @@ def __init__( Initialize AsyncDendrite with optional domain authentication. Args: - playwright_options: Options for configuring Playwright - remote_config: Remote browser provider configuration - config: Configuration object - auth: List of domains or single domain to load authentication state for + playwright_options (dict): Options for configuring Playwright browser instance. + Defaults to non-headless mode with stealth arguments. + remote_config (Optional[Providers]): Remote browser provider configuration. + Defaults to None for local browser. + config (Optional[Config]): Configuration object for the instance. + Defaults to a new Config instance. + auth (Optional[Union[List[str], str]]): List of domains or single domain + to load authentication state for. Defaults to None. """ self._impl = self._get_impl(remote_config) self._playwright_options = playwright_options @@ -195,19 +192,23 @@ async def goto( expected_page: str = "", ) -> AsyncPage: """ - Navigates to the specified URL, optionally in a new tab + Navigates to the specified URL, optionally in a new tab. Args: - url (str): The URL to navigate to. - new_tab (bool, optional): Whether to open the URL in a new tab. Defaults to False. - timeout (Optional[float], optional): The maximum time (in milliseconds) to wait for the page to load. Defaults to 15000. - expected_page (str, optional): A description of the expected page type for verification. Defaults to an empty string. + url (str): The URL to navigate to. If no protocol is specified, https:// will be added. + new_tab (bool): Whether to open the URL in a new tab. Defaults to False. + timeout (Optional[float]): The maximum time in milliseconds to wait for navigation. + Defaults to 15000ms. Navigation will continue even if timeout occurs. + expected_page (str): A description of the expected page type for verification. + If provided, will verify the loaded page matches the description. + Defaults to empty string (no verification). Returns: AsyncPage: The page object after navigation. Raises: - Exception: If there is an error during navigation or if the expected page type is not found. + IncorrectOutcomeError: If expected_page is provided and the loaded page + doesn't match the expected description. """ # Check if the URL has a protocol if not re.match(r"^\w+://", url): @@ -245,8 +246,13 @@ async def scroll_to_bottom( """ Scrolls to the bottom of the current page. - Returns: - None + Args: + timeout (float): Maximum time in milliseconds to attempt scrolling. + Defaults to 30000ms. + scroll_increment (int): Number of pixels to scroll in each step. + Defaults to 1000 pixels. + no_progress_limit (int): Number of consecutive attempts with no progress + before stopping. Defaults to 3 attempts. """ active_page = await self.get_active_page() await active_page.scroll_to_bottom( @@ -305,10 +311,12 @@ async def add_cookies(self, cookies): Adds cookies to the current browser context. Args: - cookies (List[Dict[str, Any]]): A list of cookies to be added to the browser context. + cookies (List[Dict[str, Any]]): A list of cookie objects to be added. + Each cookie should be a dictionary with standard cookie attributes + (name, value, domain, etc.). Raises: - Exception: If the browser context is not initialized. + DendriteException: If the browser context is not initialized. """ if not self.browser_context: raise DendriteException("Browser context not initialized") @@ -446,8 +454,16 @@ async def save_auth(self, url: str) -> None: """ Save authentication state for a specific domain. + This method captures and stores the current browser context's storage state + (cookies and origin data) for the specified domain. The state can be later + used to restore authentication. + Args: - domain (str): Domain to save authentication for (e.g., "github.com") + url (str): URL or domain to save authentication for (e.g., "github.com" + or "https://github.com"). The domain will be extracted from the URL. + + Raises: + DendriteException: If the browser context is not initialized. """ if not self.browser_context: raise DendriteException("Browser context not initialized") @@ -482,11 +498,15 @@ async def setup_auth( message: str = "Please log in to the website. Once done, press Enter to continue...", ) -> None: """ - Set up authentication for a specific URL. + Set up authentication for a specific URL by guiding the user through login. + + This method opens a browser window, navigates to the specified URL, waits for + the user to complete the login process, and then saves the authentication state. Args: url (str): URL to navigate to for login - message (str): Message to show while waiting for user input + message (str): Message to show while waiting for user to complete login. + Defaults to standard login instruction message. """ # Extract domain from URL # domain = urlparse(url).netloc diff --git a/dendrite/browser/sync_api/dendrite_browser.py b/dendrite/browser/sync_api/dendrite_browser.py index 6747a19..dad3305 100644 --- a/dendrite/browser/sync_api/dendrite_browser.py +++ b/dendrite/browser/sync_api/dendrite_browser.py @@ -58,23 +58,16 @@ class Dendrite( Dendrite is a class that manages a browser instance using Playwright, allowing interactions with web pages using natural language. - This class handles initialization with API keys for Dendrite, OpenAI, and Anthropic, manages browser - contexts, and provides methods for navigation, authentication, and other browser-related tasks. + This class handles initialization with configuration options, manages browser contexts, + and provides methods for navigation, authentication, and other browser-related tasks. Attributes: - id (UUID): The unique identifier for the Dendrite instance. - auth_data (Optional[AuthSession]): The authentication session data for the browser. - dendrite_api_key (str): The API key for Dendrite, used for interactions with the Dendrite API. - playwright_options (dict): Options for configuring the Playwright browser instance. - playwright (Optional[Playwright]): The Playwright instance managing the browser. + id (str): The unique identifier for the Dendrite instance. browser_context (Optional[BrowserContext]): The current browser context, which may include cookies and other session data. active_page_manager (Optional[PageManager]): The manager responsible for handling active pages within the browser context. user_id (Optional[str]): The user ID associated with the browser session. - browser_api_client (BrowserAPIClient): The API client used for communicating with the Dendrite API. - api_config (APIConfig): The configuration for the language models, including API keys for OpenAI and Anthropic. - - Raises: - Exception: If any of the required API keys (Dendrite, OpenAI, Anthropic) are not provided or found in the environment variables. + logic_engine (LogicEngine): The engine used for processing natural language interactions. + closed (bool): Whether the browser instance has been closed. """ def __init__( @@ -88,10 +81,14 @@ def __init__( Initialize Dendrite with optional domain authentication. Args: - playwright_options: Options for configuring Playwright - remote_config: Remote browser provider configuration - config: Configuration object - auth: List of domains or single domain to load authentication state for + playwright_options (dict): Options for configuring Playwright browser instance. + Defaults to non-headless mode with stealth arguments. + remote_config (Optional[Providers]): Remote browser provider configuration. + Defaults to None for local browser. + config (Optional[Config]): Configuration object for the instance. + Defaults to a new Config instance. + auth (Optional[Union[List[str], str]]): List of domains or single domain + to load authentication state for. Defaults to None. """ self._impl = self._get_impl(remote_config) self._playwright_options = playwright_options @@ -182,19 +179,23 @@ def goto( expected_page: str = "", ) -> Page: """ - Navigates to the specified URL, optionally in a new tab + Navigates to the specified URL, optionally in a new tab. Args: - url (str): The URL to navigate to. - new_tab (bool, optional): Whether to open the URL in a new tab. Defaults to False. - timeout (Optional[float], optional): The maximum time (in milliseconds) to wait for the page to load. Defaults to 15000. - expected_page (str, optional): A description of the expected page type for verification. Defaults to an empty string. + url (str): The URL to navigate to. If no protocol is specified, https:// will be added. + new_tab (bool): Whether to open the URL in a new tab. Defaults to False. + timeout (Optional[float]): The maximum time in milliseconds to wait for navigation. + Defaults to 15000ms. Navigation will continue even if timeout occurs. + expected_page (str): A description of the expected page type for verification. + If provided, will verify the loaded page matches the description. + Defaults to empty string (no verification). Returns: Page: The page object after navigation. Raises: - Exception: If there is an error during navigation or if the expected page type is not found. + IncorrectOutcomeError: If expected_page is provided and the loaded page + doesn't match the expected description. """ if not re.match("^\\w+://", url): url = f"https://{url}" @@ -227,8 +228,13 @@ def scroll_to_bottom( """ Scrolls to the bottom of the current page. - Returns: - None + Args: + timeout (float): Maximum time in milliseconds to attempt scrolling. + Defaults to 30000ms. + scroll_increment (int): Number of pixels to scroll in each step. + Defaults to 1000 pixels. + no_progress_limit (int): Number of consecutive attempts with no progress + before stopping. Defaults to 3 attempts. """ active_page = self.get_active_page() active_page.scroll_to_bottom( @@ -276,10 +282,12 @@ def add_cookies(self, cookies): Adds cookies to the current browser context. Args: - cookies (List[Dict[str, Any]]): A list of cookies to be added to the browser context. + cookies (List[Dict[str, Any]]): A list of cookie objects to be added. + Each cookie should be a dictionary with standard cookie attributes + (name, value, domain, etc.). Raises: - Exception: If the browser context is not initialized. + DendriteException: If the browser context is not initialized. """ if not self.browser_context: raise DendriteException("Browser context not initialized") @@ -410,8 +418,16 @@ def save_auth(self, url: str) -> None: """ Save authentication state for a specific domain. + This method captures and stores the current browser context's storage state + (cookies and origin data) for the specified domain. The state can be later + used to restore authentication. + Args: - domain (str): Domain to save authentication for (e.g., "github.com") + url (str): URL or domain to save authentication for (e.g., "github.com" + or "https://github.com"). The domain will be extracted from the URL. + + Raises: + DendriteException: If the browser context is not initialized. """ if not self.browser_context: raise DendriteException("Browser context not initialized") @@ -439,11 +455,15 @@ def setup_auth( message: str = "Please log in to the website. Once done, press Enter to continue...", ) -> None: """ - Set up authentication for a specific URL. + Set up authentication for a specific URL by guiding the user through login. + + This method opens a browser window, navigates to the specified URL, waits for + the user to complete the login process, and then saves the authentication state. Args: url (str): URL to navigate to for login - message (str): Message to show while waiting for user input + message (str): Message to show while waiting for user to complete login. + Defaults to standard login instruction message. """ domain = get_domain_w_suffix(url) try: diff --git a/dendrite/logic/config.py b/dendrite/logic/config.py index b69daf7..ea6898e 100644 --- a/dendrite/logic/config.py +++ b/dendrite/logic/config.py @@ -10,6 +10,21 @@ class Config: + """ + Configuration class for Dendrite that manages file paths and LLM settings. + + This class handles the configuration of cache locations, authentication sessions, + and LLM (Language Learning Model) settings for the Dendrite system. + + Attributes: + cache_path (Path): Path to the cache directory + llm_config (LLMConfig): Configuration for language learning models + extract_cache (FileCache): Cache for extracted script data + element_cache (FileCache): Cache for element selectors + storage_cache (FileCache): Cache for browser storage states + auth_session_path (Path): Path to authentication session data + """ + def __init__( self, root_path: Union[str, Path] = ".dendrite", @@ -17,6 +32,19 @@ def __init__( auth_session_path: Union[str, Path] = "auth", llm_config: Optional[LLMConfig] = None, ): + """ + Initialize the Config with specified paths and LLM configuration. + + Args: + root_path (Union[str, Path]): Base directory for all Dendrite data. + Defaults to ".dendrite". + cache_path (Union[str, Path]): Directory name for cache storage relative + to root_path. Defaults to "cache". + auth_session_path (Union[str, Path]): Directory name for authentication + sessions relative to root_path. Defaults to "auth". + llm_config (Optional[LLMConfig]): Configuration for language models. + If None, creates a default LLMConfig instance. + """ self.cache_path = root_path / Path(cache_path) self.llm_config = llm_config or LLMConfig() self.extract_cache = FileCache(Script, self.cache_path / "extract.json") diff --git a/dendrite/logic/extract/compress_html.py b/dendrite/logic/extract/compress_html.py index f7fb3fc..b2ec20a 100644 --- a/dendrite/logic/extract/compress_html.py +++ b/dendrite/logic/extract/compress_html.py @@ -3,7 +3,7 @@ from collections import Counter from typing import List, Optional, Tuple, TypedDict, Union -from bs4 import BeautifulSoup, NavigableString, PageElement +from bs4 import BeautifulSoup, NavigableString, PageElement, Tag from bs4.element import Tag from dendrite.logic.dom.truncate import ( @@ -385,18 +385,25 @@ def traverse(tag: Union[BeautifulSoup, Tag]): # child.replace_with( # f"[...{', '.join(next_element_tags)} tags collapsed for readability...]") - def remove_double_nested(soup): - for tag in soup.find_all(True): + def remove_double_nested( + soup: Union[BeautifulSoup, Tag] + ) -> Union[BeautifulSoup, Tag]: + for tag in soup.find_all(): # If a tag only contains a single child of the same type - if len(tag.find_all(True, recursive=False)) == 1 and isinstance( - tag.contents[0], Tag + children = tag.find_all(recursive=False) + if ( + len(children) == 1 + and tag.contents + and isinstance(tag.contents[0], Tag) ): child_tag = tag.contents[0] # move the contents of the child tag up to the parent tag.clear() tag.extend(child_tag.contents) - if len(tag.find_all(True, recursive=False)) == 1 and isinstance( - tag.contents[0], Tag + if ( + len(tag.find_all(recursive=False)) == 1 + and tag.contents + and isinstance(tag.contents[0], Tag) ): remove_double_nested(tag) diff --git a/dendrite/logic/get_element/get_element.py b/dendrite/logic/get_element/get_element.py index 8e28ec7..37255c3 100644 --- a/dendrite/logic/get_element/get_element.py +++ b/dendrite/logic/get_element/get_element.py @@ -21,7 +21,7 @@ async def get_element(dto: GetElementsDTO, config: Config) -> GetElementResponse: if isinstance(dto.prompt, str): return await process_prompt(dto.prompt, dto, config) - raise ... + raise NotImplementedError("Prompt is not a string") async def process_prompt( diff --git a/dendrite/logic/llm/config.py b/dendrite/logic/llm/config.py index abc35eb..0d527b3 100644 --- a/dendrite/logic/llm/config.py +++ b/dendrite/logic/llm/config.py @@ -28,11 +28,32 @@ class LLMConfig: + """ + Configuration class for Language Learning Models (LLMs) in Dendrite. + + This class manages the registration and retrieval of different LLM agents used + throughout the system. It maintains a registry of LLM configurations for various + agents and provides a default configuration when needed. + + Attributes: + registered_llms (Dict[str, LLM]): Dictionary mapping agent names to their LLM configurations + default_llm (LLM): Default LLM configuration used when no specific agent is found + """ + def __init__( self, default_agents: Optional[Dict[str, LLM]] = None, default_llm: Optional[LLM] = None, ): + """ + Initialize the LLMConfig with optional default configurations. + + Args: + default_agents (Optional[Dict[str, LLM]]): Dictionary of agent names to LLM + configurations to override the default agents. Defaults to None. + default_llm (Optional[LLM]): Default LLM configuration to use when no + specific agent is found. If None, uses Claude 3 Sonnet with default settings. + """ self.registered_llms: Dict[str, LLM] = DEFAULT_LLM.copy() if default_agents: self.registered_llms.update(default_agents) @@ -43,20 +64,22 @@ def __init__( async def register_agent(self, agent: str, llm: LLM) -> None: """ - Register an LLM agent by name. + Register a single LLM agent configuration. Args: - agent: The name of the agent to register - llm: The LLM agent to register + agent (str): The name of the agent to register + llm (LLM): The LLM configuration to associate with the agent """ self.registered_llms[agent] = llm async def register(self, agents: Dict[str, LLM]) -> None: """ - Register multiple LLM agents at once. Overrides if an agent has already been registered + Register multiple LLM agent configurations at once. + + This method will override any existing agent configurations with the same names. Args: - agents: A dictionary of agent names to LLM agents + agents (Dict[str, LLM]): Dictionary mapping agent names to their LLM configurations """ self.registered_llms.update(agents) @@ -81,15 +104,23 @@ def get( use_default: bool = True, ) -> Optional[LLM]: """ - Get an LLM agent by name, optionally falling back to default if not found. + Get an LLM configuration by agent name. + + This method attempts to retrieve an LLM configuration in the following order: + 1. From the registered agents + 2. From the provided default parameter + 3. From the instance's default_llm (if use_default is True) Args: - agent: The name of the agent to retrieve - default: Optional specific default LLM to use if agent not found - use_default: If True, use self.default_llm when agent not found and default is None + agent (str): The name of the agent whose configuration to retrieve + default (Optional[LLM]): Specific default LLM to use if agent not found. + Defaults to None. + use_default (bool): Whether to use the instance's default_llm when agent + is not found and no specific default is provided. Defaults to True. Returns: - Optional[LLM]: The requested LLM agent, default LLM, or None + Optional[LLM]: The requested LLM configuration, or None if not found and + no defaults are available or allowed. """ llm = self.registered_llms.get(agent) if llm is not None: diff --git a/dendrite/models/api_config.py b/dendrite/models/api_config.py deleted file mode 100644 index ad58987..0000000 --- a/dendrite/models/api_config.py +++ /dev/null @@ -1,32 +0,0 @@ -from typing import Optional - -from pydantic import BaseModel, model_validator - -from dendrite.browser._common._exceptions.dendrite_exception import MissingApiKeyError - - -class APIConfig(BaseModel): - """ - Configuration model for API keys used in the Dendrite SDK. - - Attributes: - dendrite_api_key (Optional[str]): The API key for Dendrite services. - openai_api_key (Optional[str]): The API key for OpenAI services. If you wish to use your own API key, you can do so by passing it to the Dendrite. - anthropic_api_key (Optional[str]): The API key for Anthropic services. If you wish to use your own API key, you can do so by passing it to the Dendrite. - - Raises: - ValueError: If a valid dendrite_api_key is not provided. - """ - - dendrite_api_key: Optional[str] = None - openai_api_key: Optional[str] = None - anthropic_api_key: Optional[str] = None - - @model_validator(mode="before") - def _check_api_keys(cls, values): - dendrite_api_key = values.get("dendrite_api_key") - if not dendrite_api_key: - raise MissingApiKeyError( - "A valid dendrite_api_key must be provided. Make sure you have set the DENDRITE_API_KEY environment variable or passed it to the Dendrite." - ) - return values diff --git a/poetry.lock b/poetry.lock index 82b61ed..0d35e8c 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.4 and should not be changed by hand. [[package]] name = "aiohappyeyeballs" @@ -2766,4 +2766,4 @@ type = ["pytest-mypy"] [metadata] lock-version = "2.0" python-versions = "^3.9" -content-hash = "8f623a140b1f5a63f967123172463d07d41c4269d840262fd6dbcd2f4bab4b6d" +content-hash = "86cd1eefdbd4c9961104be1ceb028cdc75fd91004fb2140aadba10e653dfdea5" diff --git a/pyproject.toml b/pyproject.toml index 89801b1..eafaa47 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "dendrite" -version = "0.2.7" +version = "0.2.8" description = "Dendrite is a suite of tools that makes it easy to create web integrations for AI agents. With Dendrite your can: Authenticate on websites, Interact with elements, Extract structured data, Download and upload files, Fill out forms" authors = [ @@ -39,7 +39,7 @@ pillow = "^11.0.0" json-repair = "^0.30.1" tldextract = "^5.1.3" anthropic = "^0.42.0" - +python-dotenv = "^1.0.1" [tool.poetry.group.dev.dependencies] autopep8 = "^2.0.4"